Merged Code from: /branches/v0.5-new_cache_system/ into main trunk.

- Revisions 4771 => 5334

 * This merge brings a lot of unfinished code for GXS (new cache system)
 	- See branch commits for more details.
 * Code is disabled, and should have minimal effect on trunk build.



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@5338 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/dbase/cachestrapper.cc

@@ -279,21 +279,6 @@ bool	CacheStore::getStoredCache(CacheData &data)
 	return ok;
 }
 
-bool CacheStore::cached(const std::string /*cacheId*/)
-{
-
-	return false;
-}
-
-void CacheStore::updateCacheDocument(pugi::xml_document& /*cacheDoc*/)
-{
-	return;
-}
-
-void CacheStore::updateCacheTable()
-{
-	return;
-}
 
 bool	CacheStore::locked_getStoredCache(CacheData &data)
 {

libretroshare/src/dbase/cachestrapper.h

@@ -27,7 +27,6 @@
 #include "pqi/p3cfgmgr.h"
 #include "pqi/pqimonitor.h"
 #include "util/rsthreads.h"
-#include "util/pugixml.h"
 
 #include <list>
 #include <map>
@@ -316,33 +315,7 @@ class CacheStore
 		 */
 		bool    locked_getStoredCache(CacheData &data);
 
-
-		//////////////// Cache Optimisation //////////////////
-
-		/**
-		 *@param id the key for determing whether this data has been cached or not
-		 *@return true if data referenced by key has been cached false otherwise
-		 */
-		bool cached(const std::string cacheId);
-
-		/**
-		 * TODO: will be abstract
-		 * The deriving class should return a document which accurately reflects its data
-		 * structure
-		 * @param cacheDoc document reflecting derving class's cache data structure
-		 */
-		virtual void updateCacheDocument(pugi::xml_document& cacheDoc);
-
-
-
-
-		//////////////// Cache Optimisation //////////////////
 		private:
-		/**
-		 * This updates the cache table with information from the xml document
-		 *
-		 */
-		void updateCacheTable();
 
 		uint16_t cacheType;    /* for checking */
 		bool     multiCache;   /* do we care about subid's */
@@ -355,14 +328,6 @@ class CacheStore
 		mutable RsMutex cMutex;
 		std::map<RsPeerId, CacheSet> caches;
 
-		////////////// cache optimisation ////////////////
-
-		/// whether to run in cache optimisation mode
-		bool cacheOptMode;
-
-		/// stores whether given instance of cache data has been loaded already or not
-		std::map<std::string, bool>  cacheTable;
-
 };
 
 

libretroshare/src/gxs/UseCases.txt

@@ -0,0 +1,341 @@
+
+These are some example usages of the GxsService...
+All of these must be provided by the basic system.
+
+chris:
+Please not all request* functions have a receive pair if not noted below
+
+
+1) Forums
+------------------------------------
+1.1) Display of subscribed forums.  Service must provide:
+	1.1.1) list of subscribed Groups.
+		----------------
+                int requestSubscribedGrps();
+
+                    Component:
+                        RsGnp::requestGroups()
+
+
+		----------------
+	1.1.2) list of messages for each Group.
+                ----------------
+                int requestGrpMsgs(std::list<RsGxsGrpId>&);
+                ----------------
+
+	1.1.3) details of group by group Id.
+               ----------------
+               int requestGrp(std::list<RsGxsGrpId>& grpIds);
+               ----------------
+
+	1.1.4) details of messages by message Id.
+                ----------------
+                int requestMsgs(std::list<RsGxsMsgId>& msgs);
+                ----------------
+
+        1.1.5) newest version of each message
+
+		----------------
+                1.1.4 : not intitialising RsGxsMsgId::version default to returning
+                newest version of message
+		----------------
+
+	1.1.6) status of message (read / unread).
+
+		----------------
+                returned with msg data
+                int flagMsgRead(const RsGxsMsgId& msgId, bool read)
+
+		----------------
+
+	1.1.7) access to Identity associated with Group and Messages
+
+		----------------
+		returned with message
+		----------------
+
+	1.1.8) updates when new groups arrive.
+
+		----------------
+                virtual void notifyGroupChanged(std::list<RsGxsGrpId> grpId) = 0
+		----------------
+
+	1.1.9) updates when new messages arrive.
+		----------------
+                virtual void notifyMsgChanged(std::list<RsGxsMsgId> msgId) = 0
+		----------------
+
+	1.1.10) updates when groups / messages change.
+
+		----------------
+                1.1.8 and 1.1.9
+		----------------
+
+	1.1.11) Search forums for Keywords.
+
+		----------------
+                int requestLocalSearch(RsGxsSearch* term);
+		----------------
+
+	1.1.12) Access new (time restricted) messages.
+		----------------
+                ? explain
+		----------------
+
+1.2) Creation of new material.
+	1.2.1) ability to create Groups.
+
+		----------------
+                int pushGrp(std::set<RsGxsGroup*>& grp, std::string& profileName);
+                int pushGrp(std::set<RsGxsGroup*>& grp, RsIdentityExchangeService::IdentityType type, const std::set<std::string>& peers,
+                              const std::string& pseudonymName = "");
+
+		----------------
+
+	1.2.2) ability to create Messages.
+
+                void pushMsg(std::set<RsGxsMsg*>& msg);
+
+1.3) Deletion of forums and posts.
+	1.3.1) discard forum (can't do this currently).
+                ----------------
+                done automatically by GDService based on cutoff
+                ----------------
+
+	1.3.2) delete post.
+		----------------
+		// post is removed from system and note made in db not to syn it
+                // this is cleaned from db post expires
+                // very likely to also propagate to friends
+                int requestDeleteMsg(const RsGxsMsgId& msgId);
+                int requestDeleteGrp(const RsGxsGrpId& grpId);
+		----------------
+
+	1.3.3) discard old posts.
+		----------------
+		done automatically by GDService based on cutoff
+		----------------
+
+1.4) Control of Forums 
+	
+	1.4.1) Storage Period of subscribed forums
+
+		---------------- 		
+                void setSubscribedGroupDiscardAge(uint32_t length);
+		----------------
+
+	1.4.2) Storage Period of unsubscribed forums.
+
+		---------------- 		
+                void setUnsubscribedGrpDiscardAge(uint32_t length);
+		----------------
+
+	1.4.3) transfer unsubscribed forums and posts (yes/no)
+
+		---------------- 		
+		don't think I want this, definitely default false
+		int GXService::allUnsubscribedTransfer(bool)
+		----------------
+
+	1.4.4) modify forums we own.
+
+		---------------- 		
+		int GXService::updateGroup(multimap<grpId, version>)
+		----------------
+
+		---------------- 		
+		int GXService::updateMsg(multimap<grpId, msgId>)
+		----------------
+
+	1.4.5) modify read status.
+
+		---------------- 		
+                int flagMsgRead(const RsGxsMsgId& msgId, bool read) ;
+		----------------
+	1.4.6) Subscribe to forums
+
+		----------------
+                int requestSubscribeToGrp(const RsGxsGrpId& grpId, bool subscribe);
+		----------------
+ 		
+1.5) Finding other forums.
+	1.5.1) Listing of other forums.
+		int GXService::requestGroupList();
+		void receiveGroupList(token, list<grpId>); //call back
+	1.5.2) ability to temporarily get forum messages.
+	? Explain
+	1.5.3) search remote forums for keywords.
+		---------------- 
+		// service will have to provide own search module (can RsGDService can 
+		// service remote search requests concurrent with GXS)
+		// module return message and groups (msgs can be marked  compile time as not 		// searchable
+                int requestRemoteSearch(RsGxsSearch* term, uint32_t hops);
+
+		---------------- 
+	1.5.4) optional download of friends forums.
+		----------------
+                int requestPeersGrps(const std::string& sslId);
+		----------------
+
+
+2) Channels
+------------------------------------
+2.1) Mostly like Forums, additional functionality.
+	2.1.1) Download files associated with Post.
+
+		---------------- 
+		? Explain, This should not be part of interface, this is feature specific 
+		---------------- 
+
+2.2) Comments
+	2.2.1) Write a comment.
+		---------------- 
+                see 1.2.2. RsGxsMsg with required indirection data members
+                to associate with message the comment is being made on
+		---------------- 
+
+	2.2.2) Reject (downgrade) bad comment.
+		---------------- 
+                user needs to verify publisher sent downgrage message
+                bool RsGxis::bool verify(const KeyRef& keyref, unsigned char* data, uint32_t dataLen, std::string& signature);
+
+		---------------- 
+	
+	2.2.3) Control Comments: Allow / Disallow, Pseudo / Authed Comments. 
+		---------------- 
+                bool RsGxis::bool sign(const KeyRef& keyref, unsigned char* data, uint32_t dataLen, std::string& signature);
+
+		---------------- 
+
+
+3) Wiki
+------------------------------------
+3.1) Editing functionality
+	3.1.1) Access to Latest Version.
+
+		---------------- 
+		Pls see 1.1.7
+		---------------- 
+
+	3.1.2) Access to Version History. (including who edited).
+
+		---------------- 
+                // need to include std::map<version,authors> as top level RsGxsMsg
+		---------------- 
+
+	3.2) Controlling Editing Rights.
+
+		---------------- 
+		Pls see 1.1.7
+		---------------- 
+
+	
+	3.2.1) Rejecting an Edit.
+
+		---------------- 
+		using 1.2.2 publisher can send messages with private key signature that he 			does not accept edit 'x'
+		bool GXIService verify(Identity, data, dataLen, signature)
+		---------------- 
+
+	3.2.2) Changing Edit Rights.
+using 1.2.2, publisher sends new publish key under old private key signature, and user uses 
+
+3.3) Searching for Wikis.
+	3.3.1) Search for keyword.
+		---------------- 
+		see 1.5.3
+		---------------- 
+
+	3.3.2) Search for hashtag.
+		---------------- 
+		? Explain, anything different from keyword
+		see 1.5.3
+		---------------- 
+
+	
+	3.3.3) Search for pattern?
+		---------------- 
+		Bit more detail on search type
+		RsGxsSearch{ int searchType; const int PATTERN_TYPE, TERM_TYPE; ....}
+
+		---------------- 
+
+
+4) Wire (Twitter)
+------------------------------------
+4.1) Following Friends.
+	4.1.1) Adding Person.
+
+		----------------
+		See 1.4.6
+		---------------- 
+		
+
+4.2) Search The Wire.
+	4.2.1) Search for friends.
+	4.2.2) Search for hash tags.
+	4.2.3) Search for keywords.
+	4.2.4) Search for Retweets.
+	4.2.5) Search for replies.
+		---------------- 
+		for 4.2.* see 3.3.1
+		---------------- 
+
+5) PhotoShare.
+------------------------------------
+5.1) Controlling Access.
+	5.1.1) Friends Only.
+
+		---------------- 
+		GXService::setShareGroup(grpId, group=RsGroups::GROUP_FRIENDS)
+		---------------- 
+
+	5.1.2) Groups Only.
+
+		---------------- 
+		5.1.1
+		---------------- 
+
+	5.1.3) Everyone.
+		---------------- 
+		5.1.1 GXService::setShareGroup(grpId, group=RsGroups::GROUP_ALL)
+		---------------- 
+
+	5.1.4) Deleting Photographs.
+
+		---------------- 
+		1.3.2
+		---------------- 
+
+	5.1.4) Deleting Album.
+		---------------- 
+                1.3.2
+		---------------- 
+
+6) Acadeeb
+------------------------------------
+6.1) Review Article.
+	6.1.1) Write Review/Rate.
+		---------------- 
+		see 1.2.2 for sending message 	
+		---------------- 
+
+	6.1.2) Recommend Article.
+		---------------- 
+		see 1.2.2 for sending message 	
+		---------------- 
+	6.1.3) Get Rating of Article.
+		---------------- 
+		See 3.2.1 and 2.2.2
+		---------------- 	
+
+
+6.2) Submit Review of Article
+	6.2.1) stuff.
+	6.2.2) stuff.
+
+
+7) Tracker.
+------------------------------------
+7.1) Tracker Stuff.
+

libretroshare/src/gxs/db_acadeeb.h

@@ -0,0 +1,82 @@
+/*
+ * libretroshare/src/gxp: gxp_apps.h
+ *
+ * General Exchange Protocol interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef RS_ACADEE_H
+#define RS_ACADEE_H
+
+/*******
+ * Stores a Bibliography of Academic Articles, with links to allow you to access the actual article.
+ * The data fields, should contain enough information to 
+ *  - extract full biblio (bibtex or ris formats).
+ *  - read abstract.
+ *  - review / rating of the article
+ *  - references to similar papers, and bibliography.
+ *  - keywords, and the like.
+ * 
+ * It will be possible to have multiple identical / similar / different descriptions of the same article.
+ * The class will have to handle this: sorting and matching as best it can.
+ *
+ ****/
+
+class gxp::Paper
+{
+	/* fields from ris */
+	std::string reftype;
+	std::string journal;
+	std::string title;
+	std::string issuetitle;
+	uint32_t    volume;
+	uint32_t    issue;
+	std::string publisher;
+	std::string serialnumber;
+	std::string url;
+	std::list<std::string> authors;
+	time_t      date;
+	uint32_t    startpage;
+	uint32_t    endpage;	
+	std::string language;
+
+	std::string abstract;
+
+	// KeyWords <- go into hashtags (parent class)
+	//References & Similar Papers <- go into links (parent class)
+};
+
+
+class rsAcadee: public rsGmxp
+{
+	public:
+
+	/* we want to access the  */
+
+
+
+
+};
+
+
+#endif /* RS_GXP_H */
+
+

libretroshare/src/gxs/db_gixp.h

@@ -0,0 +1,158 @@
+/*
+ * libretroshare/src/gxp: gxp.h
+ *
+ * General Exchange Protocol interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef RS_GIXP_H
+#define RS_GIXP_H
+
+
+/************************************************************************
+ * GIXP: General Identity Exchange Protocol.
+ *
+ * As we're always running into troubles with GPG signatures... we are going to 
+ * create a layer of RSA Keys for the following properties:
+ *
+ * 1) RSA Keys can be Anonymous, Self-Signed with Pseudonym, Signed by GPG Key.
+ *	- Anonymous & Pseudonym Keys will be shared network-wide (Hop by Hop).
+	- GPG signed Keys will only be shared if we can validate the signature 
+		(providing similar behaviour to existing GPG Keys).
+	- GPG signed Keys can optionally be marked for Network-wide sharing.
+ * 2) These keys can be used anywhere, specifically in the protocols described below.
+ * 3) These keys can be used to sign, encrypt, verify & decrypt
+ * 4) Keys will never need to be directly accessed - stored in this class.
+ * 5) They will be cached locally and exchanged p2p, by pull request.
+ * 6) This class will use the generalised packet storage for efficient caching & loading.
+ * 7) Data will be stored encrypted.
+ *
+ *
+ *****/
+
+
+class gixp::key
+{
+	gxip::keyref mKeyId;
+
+	PubKey *pubKey;
+	PrivateKey *privKey; /* NULL if non-existant */
+};
+
+
+/* As we want GPG signed profiles, to be usable as PSEUDONYMS further afield,
+ * we will split them out, and distribute them seperately.
+ *
+ * So a key can have multiple profiles associated with it.
+ * - They should always have a self-signed one.
+ * - optionally have a gpg-signed one.
+ */
+
+class gixp::profile
+{
+	public:
+
+	gxip::keyref mKeyId;
+
+	std::string mName;
+	time_t      mTimestamp;   /* superseded by newer timestamps */
+	uint32_t    mProfileType; /* ANONYMOUS (no name, self-signed), PSEUDONYM (self-signed), GPG (name=gpgid, gpgsigned), REVOCATION?? */ 
+	gpp::permissions mPermissions;
+
+	gxip::signature mSignature;
+};
+
+
+class  gxip::keyref
+{
+	std::string keyId;
+}
+
+class gxip::keyrefset
+{
+	std::list<gxip::keyref> keyIds;
+}
+
+class gxip::signature
+{
+	gxip::keyref signer;
+	std::string signature;
+}
+
+class gxip::signset
+{
+	std::list<gxip::signature> signs;
+};
+
+
+/*
+ * We will pinch an idea from Amplify & Briar... of using reputations to decide
+ * if we display or distribute messages.
+ *
+ * The Reputation will be associated with the Profile - It has to be stored
+ * independently of the messages - which will not be touched.
+ *
+ * This part of the code will have to be worked out.
+ */
+
+class gxip::reputation
+{
+	gxip::keyref;
+
+	int16_t score; /* -1000 => 1000 ??? */
+	std::string comment;
+
+	gxip::signature sign;
+};
+
+
+/*******
+ * NOTES:
+ * 1) much of this is already implemented in libretroshare/src/distrib/distribsecurity.cc
+ * 2) Data types will need to be shoehorned into gxmp::signedmsg format for transport.
+ * 3) Likewise this class will sit above a gdp/gnp/gsp data handling.
+ */
+
+class p3gixp
+{
+	bool createKey(gixp::profile, bool doGpgAlso); /* fills in mKeyId, and signature */
+
+	bool haveKey(keyId);
+	bool havePrivateKey(keyId);
+	bool requestKey(keyId);
+
+	bool getProfile(keyId, gixp::profile &prof);		/* generic profile - that can be distributed far and wide */
+	bool getGpgProfile(keyId, gixp::profile &prof);		/* personal profile - (normally) only shared with friends */
+
+	bool getReputations(keyId, std::list<gxip::reputation> &reps);
+	int16_t getRepScore(keyId, uint32_t flags);
+
+	/*** process data ***/
+	bool sign(KeyId, Data, signature); 	
+	bool verify(KeyId, Data, signature);
+	bool decrypt(KeyId, Data, decryptedData);
+	bool encrypt(KeyId, Data, decryptedData);
+
+};
+
+#endif /* RS_GIXP_H */
+
+

libretroshare/src/gxs/db_gmxp.h

@@ -0,0 +1,214 @@
+/*
+ * libretroshare/src/gxp: gxp.h
+ *
+ * General Exchange Protocol interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef RS_GMXP_H
+#define RS_GMXP_H
+
+
+/************************************************************************
+ * GMXP: General Message Exchange Protocol.
+ *
+ * The existing experiences of Forums & Channels have highlighted some
+ * significant limitations of the Cache-Based exchange system. Based
+ * on this initial understandings - an improved transport system will
+ * be design to provide a generalised message exchange foundation, 
+ * which can be used to build new services... 
+ *
+ *
+ * Key Properties:
+ *
+ * 1) Message independent: Should be able to be used for Forums, Channels
+ *   Twitter, Photos, Task Tracking, Link Cloud, etc.
+ * 2) Easy to Use: Specify(Msg, Permissions, KeyId) only.
+ * 3) Efficient Network Transport. (in common with GIXP)
+ * 4) Efficient Cache System (in common with GIXP).
+ * 5) Uses Groups Feature (in common with GIXP).
+ * 6) Search Protocols. ( might need overloading at higher level).
+ *
+ *****/
+
+
+/******
+ * NOTES:
+ * 1) Based on Forum/Channel Groups.
+ * 2) Will likely need to extend to handle info for other services.
+ * 3) Perhaps a more generalised class like gxmp::msg would be best with extra data.
+ *
+ */
+
+class gxmp::group
+{
+        gxp::id grpId;
+        uint32_t grpType;       /* FORUM, CHANNEL, TWITTER, etc */
+
+        uint32_t     timestamp;
+        uint32_t     grpFlags;
+        std::string  grpName;
+        std::string  grpDesc;
+        std::string  grpCategory;
+
+        RsTlvImage   grpPixmap;
+
+        gpp::permissions grpPermissions;
+
+        gxip::keyref     adminKey;
+        gxip::keyrefset  publishKeys;
+
+        gxip::signature adminSignature;
+};
+
+
+/******
+ * NOTES:
+ * 1) This represents the base of the Unpacked Msgs, it will be overloaded by all classes
+ * that want to use the service. It is difficult to go from gxmp::msg => gxmp::signedmsg
+ * so data should be stored in the signedmsg format.
+ * 2) All services will fundamentally use data types derived from this.
+ * 3) This packet is only serialised once at post time, typically it is deserialised by all nodes.
+ */
+class gmxp::link
+{
+	uint32_t linktype;
+	gxp::id  msgId;
+}
+
+class gmxp::msg
+{
+        gxp::id groupId;
+        gxp::id msgId;
+
+        gxp::id parentId;  /* for full threading */
+        gxp::id threadId;  /* top of thread Id */
+
+        gxp::id origMsgId;  /* if a replacement msg, otherwise == msgId */
+        gxp::id replacingMsgId;  /* if a replacement msg, otherwise == NULL (for ordering) */
+
+        uint32_t msgtype;          /* FORUM, CHANNEL, EVENT, COMMENT, VOTE, etc */
+        uint32_t flags;         /* Is this needed? */
+        uint32_t timestamp;
+
+	// New extensions - put these in the generic message, so we can handle search / linking for all services.
+	std::list<std::string> hashtags;
+	std::list<gmxp::link>  linked;
+
+        gpp::permissions msgPermissions; 
+
+        gxip::signset signatures; // should this be a set, or a singleton?
+};
+
+class gmxp::group: public gxp::group
+{
+	???
+};
+
+
+/******
+ * NOTES:
+ * 1) This class will be based on p3distrib - which does most of this stuff already!
+ * 2) There are lots of virtual functions - which can be overloaded to customise behaviour.
+ *    for clarity - these have not been shown here.
+ *
+ * 3) We need to extend this class to add search functionality... so you can discover new
+ *     stuff from your friends. This will need to be an overloaded functionality as the 
+ *     search will be service specific.
+ */
+
+
+/* General Interface class which is extended by specific versions.
+ *
+ * This provides most of the generic search, and access functions.
+ * As we are going to end up with diamond style double inheritance.
+ * This function needs to be pure virtual.. so there is no disambiugation issues.
+ */
+
+class rsGmxp
+{
+	/* create content */
+       std::string createGroup(std::wstring name, std::wstring desc, uint32_t flags, unsigned char *pngImageData, uint32_t imageSize);
+       std::string publishMsg(RsDistribMsg *msg, bool personalSign);
+
+	/* indicate interest in info */
+       bool    subscribeToGroup(const std::string &grpId, bool subscribe);
+
+	/* search messages (TO DEFINE) */
+
+	/* extract messages (From p3Distrib Existing Methods) */
+
+	bool    getAllGroupList(std::list<std::string> &grpids);
+        bool    getSubscribedGroupList(std::list<std::string> &grpids);
+        bool    getPublishGroupList(std::list<std::string> &grpids);
+        void    getPopularGroupList(uint32_t popMin, uint32_t popMax, std::list<std::string> &grpids);
+
+        bool    getAllMsgList(const std::string& grpId, std::list<std::string> &msgIds);
+        bool    getParentMsgList(const std::string& grpId, 
+				const std::string& pId, std::list<std::string> &msgIds);
+        bool    getTimePeriodMsgList(const std::string& grpId, uint32_t timeMin,
+                                uint32_t timeMax, std::list<std::string> &msgIds);
+
+        GroupInfo *locked_getGroupInfo(const std::string& grpId);
+        RsDistribMsg *locked_getGroupMsg(const std::string& grpId, const std::string& msgId);
+
+        void getGrpListPubKeyAvailable(std::list<std::string>& grpList);
+
+};
+
+
+class p3gmxp
+{
+	p3gmxp(int serviceType, serialiser *);
+
+	/* create content */
+       std::string createGroup(std::wstring name, std::wstring desc, uint32_t flags, unsigned char *pngImageData, uint32_t imageSize);
+       std::string publishMsg(RsDistribMsg *msg, bool personalSign);
+
+	/* indicate interest in info */
+       bool    subscribeToGroup(const std::string &grpId, bool subscribe);
+
+	/* search messages (TO DEFINE) */
+
+	/* extract messages (From p3Distrib Existing Methods) */
+
+	bool    getAllGroupList(std::list<std::string> &grpids);
+        bool    getSubscribedGroupList(std::list<std::string> &grpids);
+        bool    getPublishGroupList(std::list<std::string> &grpids);
+        void    getPopularGroupList(uint32_t popMin, uint32_t popMax, std::list<std::string> &grpids);
+
+        bool    getAllMsgList(const std::string& grpId, std::list<std::string> &msgIds);
+        bool    getParentMsgList(const std::string& grpId, 
+				const std::string& pId, std::list<std::string> &msgIds);
+        bool    getTimePeriodMsgList(const std::string& grpId, uint32_t timeMin,
+                                uint32_t timeMax, std::list<std::string> &msgIds);
+
+        GroupInfo *locked_getGroupInfo(const std::string& grpId);
+        RsDistribMsg *locked_getGroupMsg(const std::string& grpId, const std::string& msgId);
+
+        void getGrpListPubKeyAvailable(std::list<std::string>& grpList);
+
+};
+
+
+#endif /* RS_GMXP_H */
+
+

libretroshare/src/gxs/db_gxp.h

@@ -0,0 +1,443 @@
+/*
+ * libretroshare/src/gxp: gxp.h
+ *
+ * General Exchange Protocol interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef RS_GXP_H
+#define RS_GXP_H
+
+
+
+/************************************************************************
+ * This File describes the following components for a generalised exchange protocol (GXP).
+ *
+ * GPP: General Permissions Protocol: Who is allowed access to what?
+ * GDP: General Data Protocol: High Level Interface for exchanging data.
+ * GSP: General Storage Protocol: Class implementing GDP for disk access.
+ * GNP: General Network Protocol: Class implementing GDP for network exchange.
+ *
+ * This will be the mechanism to enforce groups.
+ * No idea what the data types will look like.
+ *
+ *****/
+
+/************************************************************************
+ * GPP: General Permissions Protocol
+ *
+ * This will be the mechanism to enforce groups.
+ * No idea what the data types will look like.
+ * The real challenge here is to ensure that the permissions are universal, 
+ * so they can be baked into the messages.
+ *
+ * PUBLIC:
+ *
+ * RESTRICTED:
+ *
+ * PUBLISHER_ONLY_SHARE:
+ *
+ * GROUP: list<gpgid>
+ *
+ * GROUP: signed by list<keyid>
+ *
+ * SAME AS GROUP_DESCRIPTION: (for messages)
+ *
+ * These permissions will need to be coupled to local permissions...
+ * eg. I don't want Party Photos going to Work Collegues.
+ *
+ * The combination of both permissions will determine who things are shared with.
+ *
+ *****/
+
+class gpp::group
+{
+
+
+}
+
+class gpp::Permissions
+{
+
+
+
+};
+
+
+/************************************************************************
+ * GDP: General Data Protocol
+ *
+ * Generic Data Interface used for both GNP and GSP
+ *
+ * Permissions are handled at this level... totally generic to all messages.
+ *
+ * gxmp::signedmsg represents the "original data" container format that will
+ * be tranported and stored by GSP / GNP cmodules.
+ *****/
+
+class gxp::id
+{
+	std::string id;
+};
+
+/******
+ * NOTES:
+ * 1) Based on Forum/Channel Groups.
+ * 2) Will likely need to extend to handle info for other services.
+ * 3) Perhaps a more generalised class like gxmp::msg would be best with extra data.
+ *
+ */
+
+class gxmp::group
+{
+        gxp::id grpId;
+        uint32_t grpType;       /* FORUM, CHANNEL, TWITTER, etc */
+
+        uint32_t     timestamp;
+        uint32_t     grpFlags;
+        std::string  grpName;
+        std::string  grpDesc;
+        std::string  grpCategory;
+
+        RsTlvImage   grpPixmap;
+
+        gpp::permissions grpPermissions;
+
+        gxip::keyref     adminKey;
+        gxip::keyrefset  publishKeys;
+
+        gxip::signature adminSignature;
+};
+
+/******
+ * NOTES:
+ * 1) It is possible for the packet to be encrypted, rather than just signed.
+ *    so the uncrypted data mustn't give away too much about whats in the msg.
+ * 2) This is based on the DistribSignedMsg format.
+ * 3) Combined signatures instead of explict publish/personal signs - 
+ *    to make it more general. Expect first signature == msgId.
+ * 4) Should the permissions be only a group level, or for individual msgs as well?
+ *
+ */
+
+class gxp::signedmsg
+{
+	gxp::id groupId;  /* high level groupings, e.g. specific forum, channel, or twitter user, etc. */
+	gxp::id msgId;    /* unique msgId within that group, normally signature of the message */
+
+        uint32_t timestamp;
+        gpp::permissions grpPermissions; 
+
+        gxp::data packet; /* compressed data */
+
+        gxip::signset signatures;
+};
+
+
+class gdp::interface
+{
+	/* query for available groups & messages */
+	int listgroups(std::list<gdb::id> &grpIds);
+	/* response from listmsgs: -1 = invalid parameters, 0 = incomplete list, 1 = all known msgs */
+	int listmsgs(const gdp::id grpId, std::list<gdb::id> &msgIds, const time_t from, const time_t to, const int maxmsgs);
+
+	/* response from requestMsg: YES (available immediately), RETRIEVING (known to exist), 
+	 * IN_REQUEST (might exist), NOT_AVAILABLE (still might exist)
+	 * NB: NOT_AVAILABLE could mean that it will be retrievable later when other peers come online, or possibilly we are not allowed it. 
+	 */
+
+	int requestMsg(gdp::id groupId, gdp::id msgId, double &delay); 
+	int getMsg(gdp::id groupId, gdp::id msgId, gdp::message &msg);   
+	int storeMsg(const gdp::message &msg);
+
+	/* search interface, is it here? or next level up */
+
+};
+
+
+/************************************************************************
+ * GSP: General Storage Protocol
+ *
+ * This will be the mechanism used to store both GIXP and GMXP data.
+ *
+ * This will have a decent index system - which is loaded immediately.
+ * meaning we know if a message exists or not.
+ *
+ * We will also implement a cache system, meaning recently accessed data
+ * is available immediately.
+ *
+ *
+ * 
+ *****/
+
+class gsp::storage: public gdp::interface
+{
+	/*** gdp::iterface *****/
+	int requestMsg(gdp::id groupId, gdp::id msgId, double &delay); 
+
+	int getMsg(gdp::id groupId, gdp::id msgId, gdp::message &msg);   
+	int storeMsg(const gdp::message &msg);
+
+	int flagMsg(gxp::id grpId, gxp::id msgId, bool pleaseCache);
+
+
+	/*** IMPLEMENTATION DETAILS ****/
+
+	loadIndex();
+	loadCache();
+
+	bool isInCache(gdp::id grpId, gdp::id msgId);
+	bool isInIndex(gdp::id grpId, gdp::id msgId);	
+	void fetchToCache(gdp::id grpId, gdp::id msgId);
+
+};
+
+
+
+/************************************************************************
+ * GNP: General Network Protocol
+ *
+ * This will be the mechanism share data with peers.
+ * It will be designed and closely couple to GSP for effective data managment.
+ *
+ * This part will effectively interface with librs Network Service.
+ * This will also push message ids around - to indicate new messages.
+ *
+ * This would sit above the GSP, and push things into storage 
+ * as they come over the network
+ *
+ * 
+ *****/
+
+
+class gnp::exchange: public gdp::interface
+{
+	/*** gdp::iterface *****/
+	int requestMsg(gdp::id groupId, gdp::id msgId, double &delay); 
+
+	int getMsg(gdp::id groupId, gdp::id msgId, gdp::message &msg);   
+	int storeMsg(const gdp::message &msg);
+
+	/*** IMPLEMENTATION DETAILS ****/
+
+	/* Get/Send Messages */
+	getAvailableMsgs(gdp::id grpId, time_t from, time_t to); /* request over the network */
+	sendAvailableMsgs(std::string peerId, gdp::id grpId, time_t from, time_t to); /* send to peers */
+
+	requestMessages(std::string peerId, gdp::id grpId, std::list<gdp::id> msgIds); 	
+	sendMessages(std::string peerId, gdp::id grpId, std::list<gdp::id> msgIds);	  /* send to peer, obviously permissions have been checked first */
+
+	/* Search the network */
+
+};
+
+
+/************************************************************************
+ * GIXP: General Identity Exchange Protocol.
+ *
+ * As we're always running into troubles with GPG signatures... we are going to 
+ * create a layer of RSA Keys for the following properties:
+ *
+ * 1) RSA Keys can be Anonymous, Self-Signed with Pseudonym, Signed by GPG Key.
+ *	- Anonymous & Pseudonym Keys will be shared network-wide (Hop by Hop).
+	- GPG signed Keys will only be shared if we can validate the signature 
+		(providing similar behaviour to existing GPG Keys).
+	- GPG signed Keys can optionally be marked for Network-wide sharing.
+ * 2) These keys can be used anywhere, specifically in the protocols described below.
+ * 3) These keys can be used to sign, encrypt, verify & decrypt
+ * 4) Keys will never need to be directly accessed - stored in this class.
+ * 5) They will be cached locally and exchanged p2p, by pull request.
+ * 6) This class will use the generalised packet storage for efficient caching & loading.
+ * 7) Data will be stored encrypted.
+ *****/
+
+class gixp::key
+{
+	gxip::keyref mKeyId;
+
+	PubKey *pubKey;
+	PrivateKey *privKey; /* NULL if non-existant */
+};
+
+class gixp::profile
+{
+	public:
+
+	gxip::keyref mKeyId;
+
+	std::string mPseudonym;
+	time_t      mTimestamp;   /* superseded by newer timestamps */
+	uint32_t    mProfileType; /* ANONYMOUS (no name, self-signed), PSEUDONYM (self-signed), GPG (name=gpgname, gpgsigned), REVOCATION?? */ 
+	gpp::permissions mPermissions;
+
+	gxip::signature mSignature;
+};
+
+class  gxip::keyref
+{
+	std::string keyId;
+}
+
+class gxip::keyrefset
+{
+	std::list<gxip::keyref> keyIds;
+}
+
+class gxip::signature
+{
+	gxip::keyref signer;
+	std::string signature;
+}
+
+class gxip::signset
+{
+	std::list<gxip::signature> signs;
+};
+
+/*******
+ * NOTES:
+ * 1) much of this is already implemented in libretroshare/src/distrib/distribsecurity.cc
+ * 2) Data types will need to be shoehorned into gxmp::signedmsg format for transport.
+ * 3) Likewise this class will sit above a gdp/gnp/gsp data handling.
+ */
+
+class p3gixp
+{
+	bool createKey(gixp::profile); /* fills in mKeyId, and signature */
+
+	bool haveKey(keyId);
+	bool havePrivateKey(keyId);
+	bool requestKey(keyId);
+
+	gixp::profile getProfile(keyId);
+
+	/*** process data ***/
+	bool sign(KeyId, Data, signature); 	
+	bool verify(KeyId, Data, signature);
+	bool decrypt(KeyId, Data, decryptedData);
+	bool encrypt(KeyId, Data, decryptedData);
+
+};
+
+
+/************************************************************************
+ * GMXP: General Message Exchange Protocol.
+ *
+ * The existing experiences of Forums & Channels have highlighted some
+ * significant limitations of the Cache-Based exchange system. Based
+ * on this initial understandings - an improved transport system will
+ * be design to provide a generalised message exchange foundation, 
+ * which can be used to build new services... 
+ *
+ *
+ * Key Properties:
+ *
+ * 1) Message independent: Should be able to be used for Forums, Channels
+ *   Twitter, Photos, Task Tracking, Link Cloud, etc.
+ * 2) Easy to Use: Specify(Msg, Permissions, KeyId) only.
+ * 3) Efficient Network Transport. (in common with GIXP)
+ * 4) Efficient Cache System (in common with GIXP).
+ * 5) Uses Groups Feature (in common with GIXP).
+ * 6) Search Protocols. ( might need overloading at higher level).
+ *
+ *****/
+
+
+/******
+ * NOTES:
+ * 1) This represents the base of the Unpacked Msgs, it will be overloaded by all classes
+ * that want to use the service. It is difficult to go from gxmp::msg => gxmp::signedmsg
+ * so data should be stored in the signedmsg format.
+ * 2) All services will fundamentally use data types derived from this.
+ * 3) This packet is only serialised once at post time, typically it is deserialised be all nodes.
+ */
+
+class gxmp::msg
+{
+        gxp::id groupId;
+        gxp::id msgId;
+
+        gxp::id parentId;  /* for full threading */
+        gxp::id threadId;  /* top of thread Id */
+
+        gxp::id origMsgId;  /* if a replacement msg, otherwise == msgId */
+        gxp::id replacingMsgId;  /* if a replacement msg, otherwise == NULL (for ordering) */
+
+        uint32_t type;          /* FORUM, CHANNEL, EVENT, COMMENT, VOTE, etc */
+        uint32_t flags;         /* Is this needed? */
+        uint32_t timestamp;
+
+        gpp::permissions msgPermissions; 
+
+        gxip::signset signatures;
+};
+
+
+
+/******
+ * NOTES:
+ * 1) This class will be based on p3distrib - which does most of this stuff already!
+ * 2) There are lots of virtual functions - which can be overloaded to customise behaviour.
+ *    for clarity - these have not been shown here.
+ *
+ * 3) We need to extend this class to add search functionality... so you can discover new
+ *     stuff from your friends. This will need to be an overloaded functionality as the 
+ *     search will be service specific.
+ */
+
+
+class p3gmxp
+{
+	p3gmxp(int serviceType, serialiser *);
+
+	/* create content */
+       std::string createGroup(std::wstring name, std::wstring desc, uint32_t flags, unsigned char *pngImageData, uint32_t imageSize);
+       std::string publishMsg(RsDistribMsg *msg, bool personalSign);
+
+	/* indicate interest in info */
+       bool    subscribeToGroup(const std::string &grpId, bool subscribe);
+
+	/* search messages (TO DEFINE) */
+
+	/* extract messages (From p3Distrib Existing Methods) */
+
+	bool    getAllGroupList(std::list<std::string> &grpids);
+        bool    getSubscribedGroupList(std::list<std::string> &grpids);
+        bool    getPublishGroupList(std::list<std::string> &grpids);
+        void    getPopularGroupList(uint32_t popMin, uint32_t popMax, std::list<std::string> &grpids);
+
+        bool    getAllMsgList(const std::string& grpId, std::list<std::string> &msgIds);
+        bool    getParentMsgList(const std::string& grpId, 
+				const std::string& pId, std::list<std::string> &msgIds);
+        bool    getTimePeriodMsgList(const std::string& grpId, uint32_t timeMin,
+                                uint32_t timeMax, std::list<std::string> &msgIds);
+
+        GroupInfo *locked_getGroupInfo(const std::string& grpId);
+        RsDistribMsg *locked_getGroupMsg(const std::string& grpId, const std::string& msgId);
+
+        void getGrpListPubKeyAvailable(std::list<std::string>& grpList);
+
+};
+
+
+#endif /* RS_GXP_H */
+
+

libretroshare/src/gxs/db_gxp_apps.h

@@ -0,0 +1,385 @@
+/*
+ * libretroshare/src/gxp: gxp_apps.h
+ *
+ * General Exchange Protocol interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef RS_GXP_APPS_H
+#define RS_GXP_APPS_H
+
+
+
+/************************************************************************
+ * This File describes applications that will use the GXP protocols.
+ *
+ *****/
+
+
+/************************************************************************
+ * Forums / Channels.
+ *
+ * The existing experiences of Forums & Channels have highlighted some
+ * significant limitations of the Cache-Based exchange system. The new
+ * and improved system will be based on GMXP.
+ *
+ * Existing Issues to deal with:
+ * 1) GPG Signatures take too long to verify (GPGme/gpg.exe issue)
+ * 2) Signatures are re-verified each startup.
+ * 3) Forum Messages are broadcast to all peers - excessive traffic/disk space.
+ * 4) Impossible to Edit Messages, or Comment on Channel Posts.
+ *
+ * Most of these issues (1-3) will be dealt with via GMXP GIXP system
+ *
+ * The data structures below are closely modelled on existing types.
+ *
+ *****/
+
+class gxp::forum::msg: public gmxp::msg
+{
+	/**** PROVIDED BY PARENT ***/
+	//gxp::id groupId;
+	//gxp::id msgId;
+
+	//gxp::id parentId; 
+	//gxp::id threadId;
+
+	//gxp::id origMsgId;
+	//gxp::id replacingMsgId;
+
+	//uint32_t timestamp;
+	//uint32_t type;
+	//uint32_t flags;
+
+	//gpp::permissions msgPermissions;
+	//gxp::signset signatures;
+
+	/**** SPECIFIC FOR FORUM MSG ****/
+
+	std::string srcId;
+	std::string title;
+	std::string msg;
+
+};
+
+class gxp::channel::msg: public gmxp::msg
+{
+	/**** PROVIDED BY PARENT ***/
+	//gxp::id groupId;
+	//gxp::id msgId;
+
+	//gxp::id parentId; // NOT USED.
+	//gxp::id threadId; // NOT USED.
+
+	//gxp::id origMsgId;
+	//gxp::id replacingMsgId;
+
+	//uint32_t timestamp;
+	//uint32_t type;
+	//uint32_t flags;
+
+	//gpp::permissions msgPermissions;
+	//gxp::signset signatures;
+
+	/**** SPECIFIC FOR CHANNEL MSG ****/
+
+        std::wstring subject;
+        std::wstring message;
+
+        RsTlvFileSet attachment;
+        RsTlvImage thumbnail;
+
+};
+
+
+/************************************************************************
+ * Events.
+ *
+ * It is well known that Events & Photos are the two primary uses for
+ * Facebook. It is imperative that these are implemented in GXP.
+ *
+ *****/
+
+class gxp::events::event: public gmxp::msg
+{
+	/**** PROVIDED BY PARENT ***/
+	//gxp::id groupId;
+	//gxp::id msgId;
+
+	//gxp::id parentId; 
+	//gxp::id threadId;
+
+	//gxp::id origMsgId;
+	//gxp::id replacingMsgId;
+
+	//uint32_t timestamp;
+	//uint32_t type;
+	//uint32_t flags;
+
+	//gpp::permissions msgPermissions;
+	//gxp::signset signatures;
+
+	/**** SPECIFIC FOR EVENT MSG ****/
+
+	location
+	time
+	repeats
+	invite list
+	number of places.
+	host
+
+
+};
+
+class gxp::events::reply: public gmxp::msg
+{
+	/**** PROVIDED BY PARENT ***/
+	//gxp::id groupId;
+	//gxp::id msgId;
+
+	//gxp::id parentId; 
+	//gxp::id threadId;
+
+	//gxp::id origMsgId;
+	//gxp::id replacingMsgId;
+
+	//uint32_t timestamp;
+	//uint32_t type;
+	//uint32_t flags;
+
+	//gpp::permissions msgPermissions;
+	//gxp::signset signatures;
+
+	/**** SPECIFIC FOR REPLY MSG ****/
+
+	bool amcoming;
+
+};
+
+
+/************************************************************************
+ * Photos.
+ *
+ *****/
+
+class gxp::PhotoAlbum: public gmxp::group
+{
+	Location
+	Period
+	Type
+	Photographer
+	Comments
+};
+
+
+
+class gxp::photos::photo: public gmxp::msg
+{
+	/**** PROVIDED BY PARENT ***/
+	//gxp::id groupId;
+	//gxp::id msgId;
+
+	//gxp::id parentId; 
+	//gxp::id threadId;
+
+	//gxp::id origMsgId;
+	//gxp::id replacingMsgId;
+
+	//uint32_t timestamp;
+	//uint32_t type;
+	//uint32_t flags;
+
+	//gpp::permissions msgPermissions;
+	//gxp::signset signatures;
+
+	/**** SPECIFIC FOR PHOTO MSG ****/
+
+	Location
+	Time
+	Type
+	Photographer
+	Comment
+
+	FileLink.
+};
+
+
+/************************************************************************
+ * Wiki.
+ *
+ * Wiki pages are very similar to Forums... Just they are editable
+ * but anyone, and will replace the earlier version.
+ *
+ *****/
+
+
+class gxp::wiki::topic: public gmxp::group
+{
+
+
+}
+
+
+class gxp::wiki::page: public gmxp::msg
+{
+	/**** PROVIDED BY PARENT ***/
+	//gxp::id groupId;
+	//gxp::id msgId;
+
+	//gxp::id parentId; 
+	//gxp::id threadId;
+
+	//gxp::id origMsgId;
+	//gxp::id replacingMsgId;
+
+	//uint32_t timestamp;
+	//uint32_t type;
+	//uint32_t flags;
+
+	//gpp::permissions msgPermissions;
+	//gxp::signset signatures;
+
+	/**** SPECIFIC FOR FORUM MSG ****/
+
+	Title
+	Format
+	Page
+
+	Links
+	References
+
+};
+
+
+/************************************************************************
+ * Links.
+ *
+ *****/
+
+class gxp::Link
+{
+
+
+
+
+};
+
+
+
+/************************************************************************
+ * Comments.
+ *
+ *****/
+
+class gxp::Comment
+{
+
+
+
+
+};
+
+
+/************************************************************************
+ * Vote.
+ *
+ *****/
+
+class gxp::Vote
+{
+
+
+
+
+};
+
+
+/************************************************************************
+ * Status
+ *
+ *****/
+
+class gxp::Status
+{
+
+
+
+
+};
+
+
+
+/************************************************************************
+ * Tasks
+ *
+ *****/
+
+class gxp::Task
+{
+
+
+
+
+};
+
+
+/************************************************************************
+ * Tweet
+ *
+ *****/
+
+class gxp::Tweet
+{
+	HashTags
+
+	Content
+
+	Links
+};
+
+
+/************************************************************************
+ * Library.
+ *
+ *****/
+
+class gxp::Paper
+{
+	KeyWords
+
+	Journal
+
+	Authors
+
+	Abstract
+
+	References
+
+	Similar Papers
+};
+
+
+
+
+
+#endif /* RS_GXP_H */
+
+

libretroshare/src/gxs/db_gxp_service.h

@@ -0,0 +1,104 @@
+
+/* Generalised Service Base Class.
+ * The base class interfaces with gixp, disk, net services.
+ */
+
+class RsGxpLink
+{
+	uint32_t type;
+	std::string msgId;
+}
+
+class RsGxpItem
+{
+	std::string msgId;
+	std::string origMsgId;
+	std::string groupId;
+	gxpTime		time;
+	std::list<std::string> hashtags;
+	std::list<RsGxpLink> linked;
+
+        RsGxpSignature mSignature;
+};
+
+class RsGxpGroup
+{
+
+
+
+};
+
+class gxp_service
+{
+	public:
+
+	requestItem(msgId);
+
+	recv(RsGxpItem *item);
+	send(RsGxpItem *item);
+
+	/****************************************************************************************************/
+	// Event Callback for the service.
+	/****************************************************************************************************/
+
+	notify_groupChanged();  // Mainly for GUI display.
+	notify_newMessage(); // used for newsFeeds.	
+	notify_duplicateMessage(); // Channels needs this for Downloading stuff, can probably be moved above.
+	locked_checkDistribMsg(); // required overload?
+	/****************************************************************************************************/
+	// Must worry about sharing keys.
+	// Can gixp handle it? 
+	// What interfaces do we need here?
+	/****************************************************************************************************/
+
+	/****************************************************************************************************/
+	// Interface for Message Read Status
+	// At the moment this is overloaded to handle autodownload flags too.
+	// This is a configuration thing.
+	/****************************************************************************************************/
+
+	int flagItemRead(std::string id);
+	int flagItemUnread(std::string id);
+	int flagGroupRead(std::string id);
+	int flagGroupUnread(std::string id);
+
+	/****************************************************************************************************/
+	// External Interface for Data.
+	/****************************************************************************************************/
+
+	// Mesage Creation.
+	int createGroup(RsGxpGroup *grp);
+	int createItem(RsGxpItem *item);
+
+	// Group Lists & Message Lists.
+	int getGroupsChanged(std::list<std::string> &groupIds);
+	int getGroups(std::list<std::string> &groupIds);
+	int getGroupList(std::string grpId, std::list<std::string> &groupIds);
+	int getTimeRange(GxpTime from, GxpTime to, std::list<std::string> &msgIds);
+	int getGroupTimeRange(std::string grpId, GxpTime from, GxpTime to, std::list<std::string> &msgIds);
+	int getReplacementMsgs(std::string origId, std::list<std::string> replaceIds);
+
+	// Getting the Actual Data.
+	int haveItem(std::string msgId);
+	int requestItem(std::string msgId);
+	RsGxpItem  *getMsg_locked(std::string msgId); 
+	RsGxpGroup *getGroup_locked(std::string msgId); 
+
+	// Interface with GIXP Stuff... everything else should be behind the scenes.
+	RsGixpProfile *getProfile_locked(std::string id);
+
+	// Immediate Search...
+	int searchReferringLinks(RsGxpLink link, std::list<std::string> &msgIds);
+	int searchHashTags(std::list<std::string>, std::list<std::string> &msgIds);
+
+	// Remote Search...
+
+	private:
+
+	// Reversed Index: for hash searching.
+	std::map<std::string, std::list<ids> > mSearchMap;
+};
+
+
+
+

libretroshare/src/gxs/db_wire.h

@@ -0,0 +1,50 @@
+
+#include "gxp_service.h"
+
+class db_wire
+{
+	/* external interface for accessing the info */
+
+	/** Note this could get very busy with a large number of tweeters.
+	 * need some other method of getting data
+	 */
+
+	getTweeters(std::list<std::string> &tweeterIds);
+	{
+		getGroups();
+	}
+
+	getTweetRange(GxpTimeStamp, GxpTimeStamp, std::list<std::string> &tweetIds);
+	{
+		getTimeRange();
+	}
+
+	getTweetRangeSource(GxpTimeStamp, GxpTimeStamp, std::string tweeterId, std::list<std::string> &tweetIds);
+	{
+		getGroupTimeRange();
+	}
+
+	getTweet(std::string id, TweetData &tweet);
+	{
+		StackLock();
+
+		RsGxpItem *getMsg_locked(id);
+
+		// translate message into TweetData.
+
+	}
+
+	// Default 
+	getProfile(std::string id, TweetData &tweet);
+
+	/* returns a search code, which is used to id later delivery */
+	int searchTweets(GxpSearchCondition cond, std::list<std::string>);
+
+	/* returns a search code, which is used to id later delivery */
+	int fetchPendingSearchResults(int searchId, std::list<std::string>);
+
+	int cancelSearch(int searchId, std::list<std::string>);
+
+};
+
+

libretroshare/src/gxs/gxscoreserver.cc

@@ -0,0 +1,56 @@
+/*
+ * gxscoreserver.cpp
+ *
+ *  Created on: 24 Jul 2012
+ *      Author: crispy
+ */
+
+#include "gxscoreserver.h"
+
+GxsCoreServer::GxsCoreServer()
+{
+
+}
+
+GxsCoreServer::~GxsCoreServer()
+{
+
+	std::set<RsGxsService*>::iterator sit;
+
+	for(sit = mGxsServices.begin(); sit != mGxsServices.end(); sit++)
+		delete *sit;
+
+}
+
+
+void GxsCoreServer::run()
+{
+	std::set<RsGxsService*>::iterator sit;
+
+	double timeDelta = 0.2;
+
+	while(isRunning())
+	{
+
+#ifndef WINDOWS_SYS
+        usleep((int) (timeDelta * 1000000));
+#else
+        Sleep((int) (timeDelta * 1000));
+#endif
+
+		for(sit = mGxsServices.begin(); sit != mGxsServices.end(); sit++)
+				(*sit)->tick();
+
+	}
+}
+
+void GxsCoreServer::addService(RsGxsService* service)
+{
+	mGxsServices.insert(service);
+}
+
+bool GxsCoreServer::removeService(RsGxsService* service)
+{
+	return (mGxsServices.erase(service) > 0);
+}
+

libretroshare/src/gxs/gxscoreserver.h

@@ -0,0 +1,30 @@
+/*
+ * gxscoreserver.h
+ *
+ *  Created on: 24 Jul 2012
+ *      Author: crispy
+ */
+
+#ifndef GXSCORESERVER_H_
+#define GXSCORESERVER_H_
+
+#include "util/rsthreads.h"
+#include "gxs/rsgxs.h"
+
+class GxsCoreServer : RsThread {
+public:
+	GxsCoreServer();
+	~GxsCoreServer();
+
+	void run();
+
+	void addService(RsGxsService* service);
+	bool removeService(RsGxsService* service);
+
+private:
+
+	std::set<RsGxsService*> mGxsServices;
+	RsMutex mGxsMutex;
+};
+
+#endif /* GXSCORESERVER_H_ */

libretroshare/src/gxs/rsdataservice.cc

@@ -0,0 +1,798 @@
+#include <fstream>
+
+#include "rsdataservice.h"
+
+#define MSG_TABLE_NAME std::string("MESSAGES")
+#define GRP_TABLE_NAME std::string("GROUPS")
+
+
+// generic
+#define KEY_NXS_FILE std::string("nxsFile")
+#define KEY_NXS_FILE_OFFSET std::string("fileOffset")
+#define KEY_NXS_FILE_LEN std::string("nxsFileLen")
+#define KEY_NXS_IDENTITY std::string("identity")
+#define KEY_GRP_ID std::string("grpId")
+#define KEY_ORIG_GRP_ID std::string("origGrpId")
+#define KEY_IDENTITY_SIGN std::string("idSign")
+#define KEY_TIME_STAMP std::string("timeStamp")
+#define KEY_NXS_FLAGS std::string("flags")
+#define KEY_NXS_META std::string("meta")
+
+
+// grp table columns
+#define KEY_ADMIN_SIGN std::string("adminSign")
+#define KEY_KEY_SET std::string("keySet")
+#define KEY_GRP_NAME std::string("grpName")
+
+// grp local
+#define KEY_GRP_SUBCR_FLAG std::string("subscribeFlag")
+#define KEY_GRP_POP std::string("popularity")
+#define KEY_MSG_COUNT std::string("msgCount")
+#define KEY_GRP_STATUS std::string("grpStatus")
+#define KEY_GRP_LAST_POST std::string("lastPost")
+
+
+// msg table columns
+#define KEY_PUBLISH_SIGN std::string("publishSign")
+#define KEY_MSG_ID std::string("msgId")
+#define KEY_ORIG_MSG_ID std::string("origMsgId")
+#define KEY_MSG_PARENT_ID std::string("parentId")
+#define KEY_MSG_THREAD_ID std::string("threadId")
+#define KEY_MSG_NAME std::string("msgName")
+
+// msg local
+#define KEY_MSG_STATUS std::string("msgStatus")
+#define KEY_CHILD_TS std::string("childTs")
+
+
+
+
+/*** actual data col numbers ***/
+
+// generic
+#define COL_ACT_GROUP_ID 0
+#define COL_NXS_FILE 1
+#define COL_NXS_FILE_OFFSET 2
+#define COL_NXS_FILE_LEN 3
+#define COL_META_DATA 4
+#define COL_ACT_MSG_ID 5
+
+/*** meta column numbers ***/
+
+// grp col numbers
+
+#define COL_ADMIN_SIGN 5
+#define COL_KEY_SET 6
+#define COL_GRP_SUBCR_FLAG 7
+#define COL_GRP_POP 8
+#define COL_MSG_COUNT 9
+#define COL_GRP_STATUS 10
+#define COL_GRP_NAME 11
+#define COL_GRP_LAST_POST 12
+#define COL_ORIG_GRP_ID 13
+
+// msg col numbers
+#define COL_PUBLISH_SIGN 5
+#define COL_MSG_ID 6
+#define COL_ORIG_MSG_ID 7
+#define COL_MSG_STATUS 8
+#define COL_CHILD_TS 9
+#define COL_PARENT_ID 10
+#define COL_THREAD_ID 11
+#define COL_MSG_NAME 12
+
+// generic meta shared col numbers
+#define COL_GRP_ID 0
+#define COL_TIME_STAMP 1
+#define COL_NXS_FLAGS 2
+#define COL_IDENTITY_SIGN 3
+#define COL_IDENTITY 4
+
+#define RS_DATA_SERVICE_DEBUG
+
+RsDataService::RsDataService(const std::string &serviceDir, const std::string &dbName, uint16_t serviceType,
+                             RsGxsSearchModule *mod)
+    : RsGeneralDataService(), mServiceDir(serviceDir), mDbName(mServiceDir + "/" + dbName), mServType(serviceType){
+
+    initialise();
+
+    // for retrieving msg meta
+    msgMetaColumns.push_back(KEY_GRP_ID); msgMetaColumns.push_back(KEY_TIME_STAMP); msgMetaColumns.push_back(KEY_NXS_FLAGS);
+    msgMetaColumns.push_back(KEY_IDENTITY_SIGN); msgMetaColumns.push_back(KEY_NXS_IDENTITY); msgMetaColumns.push_back(KEY_PUBLISH_SIGN);
+    msgMetaColumns.push_back(KEY_MSG_ID); msgMetaColumns.push_back(KEY_ORIG_MSG_ID); msgMetaColumns.push_back(KEY_MSG_STATUS);
+    msgMetaColumns.push_back(KEY_CHILD_TS); msgMetaColumns.push_back(KEY_MSG_PARENT_ID); msgMetaColumns.push_back(KEY_MSG_THREAD_ID);
+    msgMetaColumns.push_back(KEY_MSG_NAME);
+
+    // for retrieving actual data
+    msgColumns.push_back(KEY_GRP_ID);  msgColumns.push_back(KEY_NXS_FILE); msgColumns.push_back(KEY_NXS_FILE_OFFSET);
+    msgColumns.push_back(KEY_NXS_FILE_LEN); msgColumns.push_back(KEY_NXS_META); msgColumns.push_back(KEY_MSG_ID);
+
+    // for retrieving grp meta data
+    grpMetaColumns.push_back(KEY_GRP_ID);  grpMetaColumns.push_back(KEY_TIME_STAMP); grpMetaColumns.push_back(KEY_NXS_FLAGS);
+    grpMetaColumns.push_back(KEY_IDENTITY_SIGN); grpMetaColumns.push_back(KEY_NXS_IDENTITY); grpMetaColumns.push_back(KEY_ADMIN_SIGN);
+    grpMetaColumns.push_back(KEY_KEY_SET); grpMetaColumns.push_back(KEY_GRP_SUBCR_FLAG); grpMetaColumns.push_back(KEY_GRP_POP);
+    grpMetaColumns.push_back(KEY_MSG_COUNT); grpMetaColumns.push_back(KEY_GRP_STATUS); grpMetaColumns.push_back(KEY_GRP_NAME);
+    grpMetaColumns.push_back(KEY_GRP_LAST_POST); grpMetaColumns.push_back(KEY_ORIG_GRP_ID);
+
+    // for retrieving actual grp data
+    grpColumns.push_back(KEY_GRP_ID); grpColumns.push_back(KEY_NXS_FILE); grpColumns.push_back(KEY_NXS_FILE_OFFSET);
+    grpColumns.push_back(KEY_NXS_FILE_LEN); grpColumns.push_back(KEY_NXS_META);
+}
+
+RsDataService::~RsDataService(){
+    mDb->closeDb();
+    delete mDb;
+}
+
+void RsDataService::initialise(){
+
+    // initialise database
+    mDb = new RetroDb(mDbName, RetroDb::OPEN_READWRITE_CREATE);
+
+    // create table for msg data
+    mDb->execSQL("CREATE TABLE " + MSG_TABLE_NAME + "(" +
+                 KEY_MSG_ID + " TEXT," +
+                 KEY_GRP_ID +  " TEXT," +
+                 KEY_NXS_FLAGS + " INT,"  +
+                 KEY_ORIG_MSG_ID +  " TEXT," +
+                 KEY_TIME_STAMP + " INT," +
+                 KEY_PUBLISH_SIGN + " BLOB," +
+                 KEY_NXS_IDENTITY + " TEXT," +
+                 KEY_IDENTITY_SIGN + " BLOB," +
+                 KEY_NXS_FILE + " TEXT,"+
+                 KEY_NXS_FILE_OFFSET + " INT," +
+                 KEY_MSG_STATUS + " INT," +
+                 KEY_CHILD_TS + " INT," +
+                 KEY_NXS_META + " BLOB," +
+                 KEY_MSG_THREAD_ID + " TEXT," +
+                 KEY_MSG_PARENT_ID + " TEXT,"+
+                 KEY_MSG_NAME + " TEXT," +
+                 KEY_NXS_FILE_LEN + " INT);");
+
+    // create table for grp data
+    mDb->execSQL("CREATE TABLE " + GRP_TABLE_NAME + "(" +
+                 KEY_GRP_ID + " TEXT," +
+                 KEY_TIME_STAMP + " INT," +
+                 KEY_ADMIN_SIGN + " BLOB," + " BLOB," +
+                 KEY_NXS_FILE + " TEXT," +
+                 KEY_NXS_FILE_OFFSET + " INT," +
+                 KEY_KEY_SET + " BLOB," +
+                 KEY_NXS_FILE_LEN + " INT," +
+                 KEY_NXS_META + " BLOB," +
+                 KEY_GRP_NAME + " TEXT," +
+                 KEY_GRP_LAST_POST + " INT," +
+                 KEY_GRP_POP + " INT," +
+                 KEY_MSG_COUNT + " INT," +
+                 KEY_GRP_SUBCR_FLAG + " INT," +
+                 KEY_GRP_STATUS + " INT," +
+                 KEY_NXS_IDENTITY + " TEXT," +
+                 KEY_ORIG_GRP_ID + " TEXT," +
+                 KEY_NXS_FLAGS + " INT," +
+                 KEY_IDENTITY_SIGN + " BLOB);");
+
+}
+
+RsGxsGrpMetaData* RsDataService::getGrpMeta(RetroCursor &c)
+{
+    RsGxsGrpMetaData* grpMeta = new RsGxsGrpMetaData();
+
+    bool ok = true;
+
+    // for extracting raw data
+    uint32_t offset = 0;
+    char* data = NULL;
+    uint32_t data_len = 0;
+
+    // grpId
+    c.getString(COL_GRP_ID, grpMeta->mGroupId);
+
+    // required definition of a group
+    ok &= !grpMeta->mGroupId.empty();
+
+
+    c.getString(COL_IDENTITY, grpMeta->mAuthorId);
+    c.getString(COL_GRP_NAME, grpMeta->mGroupName);
+    c.getString(COL_ORIG_GRP_ID, grpMeta->mOrigGrpId);
+
+    grpMeta->mPublishTs = c.getInt32(COL_TIME_STAMP);
+    grpMeta->mGroupFlags = c.getInt32(COL_NXS_FLAGS);
+
+    // identity if any
+    if(!grpMeta->mAuthorId.empty() && ok){
+        offset = 0;
+        data = (char*)c.getData(COL_IDENTITY_SIGN, data_len);
+        if(data)
+            grpMeta->idSign.GetTlv(data, data_len, &offset);
+    }
+
+    offset = 0;
+    data = (char*)c.getData(COL_ADMIN_SIGN, data_len);
+    if(data)
+        grpMeta->adminSign.GetTlv(data, data_len, &offset);
+
+
+    offset = 0; data = NULL; data_len = 0;
+    data = (char*)c.getData(COL_KEY_SET, data_len);
+
+    if(data)
+        ok &= grpMeta->keys.GetTlv(data, data_len, &offset);
+
+    // local meta
+    grpMeta->mSubscribeFlags = c.getInt32(COL_GRP_SUBCR_FLAG);
+    grpMeta->mPop = c.getInt32(COL_GRP_POP);
+    grpMeta->mMsgCount = c.getInt32(COL_MSG_COUNT);
+    grpMeta->mLastPost = c.getInt32(COL_GRP_LAST_POST);
+    grpMeta->mGroupStatus = c.getInt32(COL_GRP_STATUS);
+
+
+    if(ok)
+        return grpMeta;
+    else
+        delete grpMeta;
+
+    return NULL;
+}
+
+RsNxsGrp* RsDataService::getGroup(RetroCursor &c)
+{
+    /*!
+     * grpId, pub admin and pub publish key
+     * necessary for successful group
+     */
+    RsNxsGrp* grp = new RsNxsGrp(mServType);
+    bool ok = true;
+
+    // for manipulating raw data
+    uint32_t offset = 0;
+    char* data = NULL;
+    uint32_t data_len = 0;
+
+    // grpId
+    c.getString(COL_ACT_GROUP_ID, grp->grpId);
+    ok &= !grp->grpId.empty();
+
+    offset = 0; data_len = 0;
+    if(ok){
+
+        data = (char*)c.getData(COL_META_DATA, data_len);
+        if(data)
+            grp->meta.GetTlv(data, data_len, &offset);
+    }
+
+    /* now retrieve grp data from file */
+    std::string grpFile;
+    c.getString(COL_NXS_FILE, grpFile);
+    ok &= !grpFile.empty();
+
+    if(ok){
+
+        data_len = c.getInt32(COL_NXS_FILE_LEN);
+        offset = c.getInt32(COL_NXS_FILE_OFFSET);
+
+        char grp_data[data_len];
+        std::ifstream istrm(grpFile.c_str(), std::ios::binary);
+        istrm.seekg(offset, std::ios::beg);
+        istrm.read(grp_data, data_len);
+
+        istrm.close();
+        offset = 0;
+        ok &= grp->grp.GetTlv(grp_data, data_len, &offset);
+    }
+
+    if(ok)
+        return grp;
+    else
+        delete grp;
+
+    return NULL;
+}
+
+RsGxsMsgMetaData* RsDataService::getMsgMeta(RetroCursor &c)
+{
+
+    RsGxsMsgMetaData* msgMeta = new RsGxsMsgMetaData();
+
+    bool ok = true;
+    uint32_t data_len = 0,
+    offset = 0;
+    char* data = NULL;
+
+
+    c.getString(COL_GRP_ID, msgMeta->mGroupId);
+    c.getString(COL_MSG_ID, msgMeta->mMsgId);
+
+    // without these, a msg is meaningless
+    ok &= (!msgMeta->mGroupId.empty()) && (!msgMeta->mMsgId.empty());
+
+    c.getString(COL_ORIG_MSG_ID, msgMeta->mOrigMsgId);
+    c.getString(COL_IDENTITY, msgMeta->mAuthorId);
+    c.getString(COL_MSG_NAME, msgMeta->mMsgName);
+
+    if(!msgMeta->mAuthorId.empty()){
+        offset = 0;
+        data = (char*)c.getData(COL_IDENTITY_SIGN, data_len);
+        msgMeta->idSign.GetTlv(data, data_len, &offset);
+    }
+
+    msgMeta->mMsgFlags = c.getInt32(COL_NXS_FLAGS);
+    msgMeta->mPublishTs = c.getInt32(COL_TIME_STAMP);
+
+    offset = 0; data_len = 0;
+
+    if(ok){
+
+        data = (char*)c.getData(COL_PUBLISH_SIGN, data_len);
+        if(data)
+            msgMeta->pubSign.GetTlv(data, data_len, &offset);
+
+    }
+
+    // thread and parent id
+    c.getString(COL_THREAD_ID, msgMeta->mThreadId);
+    c.getString(COL_PARENT_ID, msgMeta->mParentId);
+
+    // local meta
+    msgMeta->mMsgStatus = c.getInt32(COL_MSG_STATUS);
+    msgMeta->mChildTs = c.getInt32(COL_CHILD_TS);
+
+    if(ok)
+        return msgMeta;
+    else
+        delete msgMeta;
+
+    return NULL;
+}
+
+
+
+RsNxsMsg* RsDataService::getMessage(RetroCursor &c)
+{
+
+    RsNxsMsg* msg = new RsNxsMsg(mServType);
+
+    bool ok = true;
+    uint32_t data_len = 0,
+    offset = 0;
+    char* data = NULL;
+    c.getString(COL_ACT_GROUP_ID, msg->grpId);
+    c.getString(COL_ACT_MSG_ID, msg->msgId);
+
+    ok &= (!msg->grpId.empty()) && (!msg->msgId.empty());
+
+    offset = 0; data_len = 0;
+    if(ok){
+
+        data = (char*)c.getData(COL_META_DATA, data_len);
+        if(data)
+            msg->meta.GetTlv(data, data_len, &offset);
+    }
+
+    /* now retrieve grp data from file */
+    std::string msgFile;
+    c.getString(COL_NXS_FILE, msgFile);
+    offset = c.getInt32(COL_NXS_FILE_OFFSET);
+    data_len = c.getInt32(COL_NXS_FILE_LEN);
+    ok &= !msgFile.empty();
+
+    if(ok){
+
+        char msg_data[data_len];
+        std::ifstream istrm(msgFile.c_str(), std::ios::binary);
+        istrm.seekg(offset, std::ios::beg);
+        istrm.read(msg_data, data_len);
+
+        istrm.close();
+        offset = 0;
+        ok &= msg->msg.GetTlv(msg_data, data_len, &offset);
+    }
+
+    if(ok)
+        return msg;
+    else
+        delete msg;
+
+    return NULL;
+}
+
+int RsDataService::storeMessage(std::map<RsNxsMsg *, RsGxsMsgMetaData *> &msg)
+{
+    std::map<RsNxsMsg*, RsGxsMsgMetaData* >::iterator mit = msg.begin();
+
+    // start a transaction
+    mDb->execSQL("BEGIN;");
+
+    for(; mit != msg.end(); mit++){
+
+        RsNxsMsg* msgPtr = mit->first;
+        RsGxsMsgMetaData* msgMetaPtr = mit->second;
+
+        // create or access file in binary
+        std::string msgFile = mServiceDir + "/" + msgPtr->grpId + "-msgs";
+        std::fstream ostrm(msgFile.c_str(), std::ios::binary | std::ios::app | std::ios::out);
+        ostrm.seekg(0, std::ios::end); // go to end to append
+        uint32_t offset = ostrm.tellg(); // get fill offset
+
+        ContentValue cv;
+
+        cv.put(KEY_NXS_FILE_OFFSET, (int32_t)offset);
+        cv.put(KEY_NXS_FILE, msgFile);
+        cv.put(KEY_NXS_FILE_LEN, (int32_t)msgPtr->msg.TlvSize());
+        cv.put(KEY_MSG_ID, msgMetaPtr->mMsgId);
+        cv.put(KEY_GRP_ID, msgMetaPtr->mGroupId);
+        char pubSignData[msgMetaPtr->pubSign.TlvSize()];
+        offset = 0;
+        msgMetaPtr->pubSign.SetTlv(pubSignData, msgMetaPtr->pubSign.TlvSize(), &offset);
+        cv.put(KEY_PUBLISH_SIGN, msgMetaPtr->pubSign.TlvSize(), pubSignData);
+
+
+        if(! (msgMetaPtr->mAuthorId.empty()) ){
+            char idSignData[msgMetaPtr->idSign.TlvSize()];
+            offset = 0;
+            msgMetaPtr->idSign.SetTlv(idSignData, msgMetaPtr->idSign.TlvSize(), &offset);
+            cv.put(KEY_IDENTITY_SIGN, msgMetaPtr->idSign.TlvSize(), idSignData);
+            cv.put(KEY_NXS_IDENTITY, msgMetaPtr->mAuthorId);
+        }
+
+        cv.put(KEY_NXS_FLAGS, (int32_t) msgMetaPtr->mMsgFlags);
+        cv.put(KEY_TIME_STAMP, (int32_t) msgMetaPtr->mPublishTs);
+
+        offset = 0;
+        char metaData[msgPtr->meta.TlvSize()];
+        msgPtr->meta.SetTlv(metaData, msgPtr->meta.TlvSize(), &offset);
+        cv.put(KEY_NXS_META, msgPtr->meta.TlvSize(), metaData);
+
+        cv.put(KEY_MSG_PARENT_ID, msgMetaPtr->mParentId);
+        cv.put(KEY_MSG_THREAD_ID, msgMetaPtr->mThreadId);
+        cv.put(KEY_ORIG_MSG_ID, msgMetaPtr->mOrigMsgId);
+        cv.put(KEY_MSG_NAME, msgMetaPtr->mMsgName);
+
+        // now local meta
+        cv.put(KEY_MSG_STATUS, (int32_t)msgMetaPtr->mMsgStatus);
+        cv.put(KEY_CHILD_TS, (int32_t)msgMetaPtr->mChildTs);
+
+        offset = 0;
+        char msgData[msgPtr->msg.TlvSize()];
+        msgPtr->msg.SetTlv(msgData, msgPtr->msg.TlvSize(), &offset);
+        ostrm.write(msgData, msgPtr->msg.TlvSize());
+        ostrm.close();
+
+        mDb->sqlInsert(MSG_TABLE_NAME, "", cv);
+    }
+
+    // finish transaction
+    return mDb->execSQL("COMMIT;");
+}
+
+
+int RsDataService::storeGroup(std::map<RsNxsGrp *, RsGxsGrpMetaData *> &grp)
+{
+    std::map<RsNxsGrp*, RsGxsGrpMetaData* >::iterator sit = grp.begin();
+
+    // begin transaction
+    mDb->execSQL("BEGIN;");
+
+    for(; sit != grp.end(); sit++){
+
+        RsNxsGrp* grpPtr = sit->first;
+        RsGxsGrpMetaData* grpMetaPtr = sit->second;
+
+        std::string grpFile = mServiceDir + "/" + grpPtr->grpId;
+        std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::app | std::ios::out);
+        ostrm.seekg(0, std::ios::end); // go to end to append
+        uint32_t offset = ostrm.tellg(); // get fill offset
+
+        /*!
+         * STORE file offset, file length, file name,
+         * grpId, flags, publish time stamp, identity,
+         * id signature, admin signatue, key set, last posting ts
+         * and meta data
+         **/
+        ContentValue cv;
+        cv.put(KEY_NXS_FILE_OFFSET, (int32_t)offset);
+        cv.put(KEY_NXS_FILE_LEN, (int32_t)grpPtr->grp.TlvSize());
+        cv.put(KEY_NXS_FILE, grpFile);
+        cv.put(KEY_GRP_ID, grpPtr->grpId);
+        cv.put(KEY_GRP_NAME, grpMetaPtr->mGroupName);
+        cv.put(KEY_ORIG_GRP_ID, grpMetaPtr->mOrigGrpId);
+        cv.put(KEY_NXS_FLAGS, (int32_t)grpMetaPtr->mGroupFlags);
+        cv.put(KEY_TIME_STAMP, (int32_t)grpMetaPtr->mPublishTs);
+
+        if(! (grpMetaPtr->mAuthorId.empty()) ){
+            cv.put(KEY_NXS_IDENTITY, grpMetaPtr->mAuthorId);
+
+            char idSignData[grpMetaPtr->idSign.TlvSize()];
+            offset = 0;
+            grpMetaPtr->idSign.SetTlv(idSignData, grpMetaPtr->idSign.TlvSize(), &offset);
+            cv.put(KEY_IDENTITY_SIGN, grpMetaPtr->idSign.TlvSize(), idSignData);
+            std::string wat(idSignData, grpMetaPtr->idSign.TlvSize());
+            std::cerr << wat << std::endl;
+        }
+
+        char adminSignData[grpMetaPtr->adminSign.TlvSize()];
+        offset = 0;
+        grpMetaPtr->adminSign.SetTlv(adminSignData, grpMetaPtr->adminSign.TlvSize(), &offset);
+        cv.put(KEY_ADMIN_SIGN, grpMetaPtr->adminSign.TlvSize(), adminSignData);
+
+        offset = 0;
+        char keySetData[grpMetaPtr->keys.TlvSize()];
+        grpMetaPtr->keys.SetTlv(keySetData, grpMetaPtr->keys.TlvSize(), &offset);
+        cv.put(KEY_KEY_SET, grpMetaPtr->keys.TlvSize(), keySetData);
+
+        offset = 0;
+        char metaData[grpPtr->meta.TlvSize()];
+        grpPtr->meta.SetTlv(metaData, grpPtr->meta.TlvSize(), &offset);
+        cv.put(KEY_NXS_META, grpPtr->meta.TlvSize(), metaData);
+
+        // local meta data
+        cv.put(KEY_GRP_SUBCR_FLAG, (int32_t)grpMetaPtr->mSubscribeFlags);
+        cv.put(KEY_GRP_POP, (int32_t)grpMetaPtr->mPop);
+        cv.put(KEY_MSG_COUNT, (int32_t)grpMetaPtr->mMsgCount);
+        cv.put(KEY_GRP_STATUS, (int32_t)grpMetaPtr->mGroupStatus);
+        cv.put(KEY_GRP_LAST_POST, (int32_t)grpMetaPtr->mLastPost);
+
+        offset = 0;
+        char grpData[grpPtr->grp.TlvSize()];
+        grpPtr->grp.SetTlv(grpData, grpPtr->grp.TlvSize(), &offset);
+        ostrm.write(grpData, grpPtr->grp.TlvSize());
+        ostrm.close();
+
+        mDb->sqlInsert(GRP_TABLE_NAME, "", cv);
+    }
+    // finish transaction
+    return mDb->execSQL("COMMIT;");
+}
+
+int RsDataService::retrieveNxsGrps(std::map<std::string, RsNxsGrp *> &grp, bool withMeta, bool cache){
+
+	if(grp.empty()){
+
+		RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpColumns, "", "");
+
+		if(c)
+		{
+			std::vector<RsNxsGrp*> grps;
+
+			retrieveGroups(c, grps, withMeta);
+			std::vector<RsNxsGrp*>::iterator vit = grps.begin();
+
+			for(; vit != grps.end(); vit++)
+			{
+				grp[(*vit)->grpId] = *vit;
+			}
+
+			delete c;
+		}
+
+	}else{
+
+		std::map<std::string, RsNxsGrp *>::iterator mit = grp.begin();
+
+		for(; mit != grp.end(); mit++)
+		{
+			const std::string& grpId = mit->first;
+			RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpColumns, "grpId='" + grpId + "'", "");
+
+			if(c)
+			{
+				std::vector<RsNxsGrp*> grps;
+				retrieveGroups(c, grps);
+
+				if(!grps.empty())
+				{
+					RsNxsGrp* ng = grps.front();
+					grp[ng->grpId] = ng;
+				}else{
+					grp.erase(grpId);
+				}
+
+				delete c;
+			}
+		}
+	}
+
+	return 1;
+}
+
+void RsDataService::retrieveGroups(RetroCursor* c, std::vector<RsNxsGrp*>& grps, bool withMeta){
+
+    if(c){
+        bool valid = c->moveToFirst();
+
+        while(valid){
+            RsNxsGrp* g = getGroup(*c);
+
+            // only add the latest grp info
+            if(g)
+            {
+            	RsGxsGrpMetaData* meta;
+
+            	if(withMeta)
+            		meta = getGrpMeta(*c);
+
+            	if(meta) g->metaData = meta;
+                grps.push_back(g);
+            }
+            valid = c->moveToNext();
+        }
+    }
+}
+
+int RsDataService::retrieveNxsMsgs(const GxsMsgReq &reqIds, GxsMsgResult &msg, bool cache)
+{
+
+    GxsMsgReq::const_iterator mit = reqIds.begin();
+
+    for(; mit != reqIds.end(); mit++)
+    {
+
+        const std::string& grpId = mit->first;
+
+        // if vector empty then request all messages
+        const std::vector<std::string>& msgIdV = mit->second;
+        std::vector<RsNxsMsg*> msgSet;
+
+        if(msgIdV.empty()){
+            RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, msgColumns, KEY_GRP_ID+ "='" + grpId + "'", "");
+
+            if(c)
+                retrieveMessages(c, msgSet);
+
+            delete c;
+        }else{
+
+            // request each grp
+            std::vector<std::string>::const_iterator sit = msgIdV.begin();
+
+            for(; sit!=msgIdV.end();sit++){
+                const std::string& msgId = *sit;
+                RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, msgColumns, KEY_GRP_ID+ "='" + grpId
+                                               + "' AND " + KEY_MSG_ID + "='" + msgId + "'", "");
+
+                if(c)
+                    retrieveMessages(c, msgSet);
+
+                delete c;
+            }
+        }
+
+        msg[grpId] = msgSet;
+        msgSet.clear();
+    }
+}
+
+void RsDataService::retrieveMessages(RetroCursor *c, std::vector<RsNxsMsg *> &msgs)
+{
+    bool valid = c->moveToFirst();
+    while(valid){
+        RsNxsMsg* m = getMessage(*c);
+
+        if(m){
+            msgs.push_back(m);;
+        }else{
+            delete m;
+        }
+
+        valid = c->moveToNext();
+    }
+    return;
+}
+
+int RsDataService::retrieveGxsMsgMetaData(const std::vector<std::string> &grpIds, GxsMsgMetaResult &msgMeta)
+{
+    std::vector<std::string>::const_iterator vit = grpIds.begin();
+
+    for(; vit != grpIds.end(); vit++)
+    {
+        const std::string& grpId = *vit;
+
+        std::vector<RsGxsMsgMetaData*> meta;
+
+        RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, msgMetaColumns, "", KEY_GRP_ID+ "='" + grpId + "'");
+
+        if(c)
+        {
+            bool valid = c->moveToFirst();
+            while(valid){
+                RsGxsMsgMetaData* m = getMsgMeta(*c);
+
+                if(m){
+                    meta.push_back(m);
+                }else{
+                    delete m;
+                }
+                valid = c->moveToNext();
+            }
+
+            msgMeta[grpId] = meta;
+        }
+        delete c;
+
+    }
+    return 1;
+}
+
+int RsDataService::retrieveGxsGrpMetaData(std::map<std::string, RsGxsGrpMetaData *>& grp)
+{
+    RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpMetaColumns, "", "");
+
+    if(c)
+    {
+        bool valid = c->moveToFirst();
+
+        while(valid)
+        {
+            RsGxsGrpMetaData* g = getGrpMeta(*c);
+
+            if(g)
+            {
+                grp[g->mGroupId] = g;
+            }
+            valid = c->moveToNext();
+        }
+    }
+    delete c;
+
+    return 1;
+}
+
+int RsDataService::resetDataStore()
+{
+
+#ifdef RS_DATA_SERVICE_DEBUG
+    std::cerr << "resetDataStore() " << std::endl;
+#endif
+
+    std::map<std::string, RsNxsGrp*> grps;
+
+    retrieveNxsGrps(grps, false, false);
+    std::map<std::string, RsNxsGrp*>::iterator mit
+            = grps.begin();
+
+    // remove all grp msgs files from service dir
+    for(; mit != grps.end(); mit++){
+        std::string file = mServiceDir + "/" + mit->first;
+        std::string msgFile = file + "-msgs";
+        remove(file.c_str()); // remove group file
+        remove(msgFile.c_str()); // and remove messages file
+    }
+
+    mDb->closeDb();
+    remove(mDbName.c_str()); // remove db file
+
+    // recreate database
+    initialise();
+
+    return 1;
+}
+
+int RsDataService::removeGroups(const std::vector<std::string> &grpIds)
+{
+
+    return 0;
+}
+
+int RsDataService::updateGroupMetaData(GrpLocMetaData *meta)
+{
+    return 0;
+}
+
+int RsDataService::updateMessageMetaData(MsgLocMetaData *metaData)
+{
+    return 0;
+}
+
+
+int RsDataService::removeMsgs(const std::string grpId, const std::vector<std::string> &msgIds)
+{
+    return 0;
+}
+
+uint32_t RsDataService::cacheSize() const {
+    return 0;
+}
+
+int RsDataService::setCacheSize(uint32_t size)
+{
+    return 0;
+}
+

libretroshare/src/gxs/rsdataservice.h

@@ -0,0 +1,171 @@
+#ifndef RSDATASERVICE_H
+#define RSDATASERVICE_H
+
+#include "gxs/rsgds.h"
+#include "util/retrodb.h"
+
+
+class RsDataService : public RsGeneralDataService
+{
+public:
+
+    RsDataService(const std::string& serviceDir, const std::string& dbName, uint16_t serviceType, RsGxsSearchModule* mod = NULL);
+    virtual ~RsDataService() ;
+
+    /*!
+     * Retrieves all msgs
+     * @param reqIds requested msg ids (grpId,msgId), leave msg list empty to get all msgs for the grp
+     * @param msg result of msg retrieval
+     * @param cache whether to store results of this retrieval in memory for faster later retrieval
+     * @return error code
+     */
+    int retrieveNxsMsgs(const GxsMsgReq& reqIds, GxsMsgResult& msg, bool cache);
+
+    /*!
+     * Retrieves groups, if empty, retrieves all grps, if map is not empty
+     * only retrieve entries, if entry cannot be found, it is removed from map
+     * @param grp retrieved groups
+     * @param withMeta this initialise the metaData member of the nxsgroups retrieved
+     * @param cache whether to store retrieval in mem for faster later retrieval
+     * @return error code
+     */
+    int retrieveNxsGrps(std::map<std::string, RsNxsGrp*>& grp, bool withMeta, bool cache);
+
+    /*!
+     * Retrieves meta data of all groups stored (most current versions only)
+     * @param cache whether to store retrieval in mem for faster later retrieval
+     * @return error code
+     */
+    int retrieveGxsGrpMetaData(std::map<std::string, RsGxsGrpMetaData*>& grp);
+
+    /*!
+     * Retrieves meta data of all groups stored (most current versions only)
+     * @param grpIds grpIds for which to retrieve meta data
+     * @param msgMeta meta data result as map of grpIds to array of metadata for that grpId
+     * @param cache whether to store retrieval in mem for faster later retrieval
+     * @return error code
+     */
+    int retrieveGxsMsgMetaData(const std::vector<std::string>& grpIds, GxsMsgMetaResult& msgMeta);
+
+    /*!
+     * remove msgs in data store
+     * @param grpId group Id of message to be removed
+     * @param msgIds ids of messages to be removed
+     * @return error code
+     */
+    int removeMsgs(const std::string grpId, const std::vector<std::string>& msgIds);
+
+    /*!
+     * remove groups in data store listed in grpIds param
+     * @param grpIds ids of groups to be removed
+     * @return error code
+     */
+    int removeGroups(const std::vector<std::string>& grpIds);
+
+    /*!
+     * @return the cache size set for this RsGeneralDataService in bytes
+     */
+    uint32_t cacheSize() const;
+
+    /*!
+     * @param size size of cache to set in bytes
+     */
+    int setCacheSize(uint32_t size);
+
+    /*!
+     * Stores a list of signed messages into data store
+     * @param msg map of message and decoded meta data information
+     * @return error code
+     */
+    int storeMessage(std::map<RsNxsMsg*, RsGxsMsgMetaData*>& msg);
+
+    /*!
+     * Stores a list of groups in data store
+     * @param grp map of group and decoded meta data
+     * @return error code
+     */
+    int storeGroup(std::map<RsNxsGrp*, RsGxsGrpMetaData*>& grp);
+
+    /*!
+     * @param metaData The meta data item to update
+     * @return error code
+     */
+    int updateMessageMetaData(MsgLocMetaData* metaData);
+
+    /*!
+     * @param metaData The meta data item to update
+     * @return error code
+     */
+    int updateGroupMetaData(GrpLocMetaData* meta);
+
+
+    /*!
+     * Completely clear out data stored in
+     * and returns this to a state
+     * as it was when first constructed
+     * @return error code
+     */
+    int resetDataStore();
+
+private:
+
+    /*!
+     * Retrieves all the msg results from a cursor
+     * @param c cursor to result set
+     * @param msgs messages retrieved from cursor are stored here
+     */
+    void retrieveMessages(RetroCursor* c, std::vector<RsNxsMsg*>& msgs);
+
+    /*!
+	 * Retrieves all the msg results from a cursor
+	 * @param c cursor to result set
+	 * @param msgs messages retrieved from cursor are stored here
+	 */
+	void retrieveGroups(RetroCursor* c, std::vector<RsNxsGrp*>& grps, bool withMeta = false);
+
+    /*!
+     * extracts a msg meta item from a cursor at its
+     * current position
+     */
+    RsGxsMsgMetaData* getMsgMeta(RetroCursor& c);
+
+    /*!
+     * extracts a grp meta item from a cursor at its
+     * current position
+     */
+    RsGxsGrpMetaData* getGrpMeta(RetroCursor& c);
+
+    /*!
+     * extracts a msg item from a cursor at its
+     * current position
+     */
+    RsNxsMsg* getMessage(RetroCursor& c);
+
+    /*!
+     * extracts a grp item from a cursor at its
+     * current position
+     */
+    RsNxsGrp* getGroup(RetroCursor& c);
+
+    /*!
+     * Creates an sql database and its associated file
+     * also creates the message and groups table
+     */
+    void initialise();
+
+private:
+
+
+    RetroDb* mDb;
+
+    std::list<std::string> msgColumns;
+    std::list<std::string> msgMetaColumns;
+
+    std::list<std::string> grpColumns;
+    std::list<std::string> grpMetaColumns;
+
+    std::string mServiceDir, mDbName;
+    uint16_t mServType;
+};
+
+#endif // RSDATASERVICE_H

libretroshare/src/gxs/rsgds.h

@@ -0,0 +1,196 @@
+#ifndef RSGDS_H
+#define RSGDS_H
+
+/*
+ * libretroshare/src/gxp: gxp.h
+ *
+ * General Data service, interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie, Evi-Parker Christopher
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <set>
+#include <map>
+#include <string>
+
+#include "inttypes.h"
+
+#include "serialiser/rsgxsitems.h"
+#include "serialiser/rsnxsitems.h"
+#include "gxs/rsgxsdata.h"
+
+
+class RsGxsSearchModule  {
+
+public:
+
+	virtual ~RsGxsSearchModule();
+
+
+};
+
+/*!
+ * This allows modification of local
+ * meta data items of a message
+ */
+class MsgLocMetaData {
+
+};
+
+/*!
+ * This allows modification of local
+ * meta data items of a group
+ */
+class GrpLocMetaData {
+
+};
+
+typedef std::map<RsGxsGroupId, std::vector<RsGxsMessageId> > GxsMsgReq; // <grpId, msgIds>
+typedef std::map<RsGxsGroupId, std::vector<RsNxsMsg*> > GxsMsgResult; // <grpId, msgs>
+typedef std::map<RsGxsGroupId, std::vector<RsGxsMsgMetaData*> > GxsMsgMetaResult; // <grpId, msg metadatas>
+
+/*!
+ * The main role of GDS is the preparation and handing out of messages requested from
+ * RsGeneralExchangeService and RsGeneralExchangeService
+ * It is important to note that no actual messages are passed by this interface as its is expected
+ * architecturally to pass messages to the service via a call back
+ *
+ * It also acts as a layer between RsGeneralStorageService and its parent RsGeneralExchangeService
+ * thus allowing for non-blocking requests, etc.
+ * It also provides caching ability
+ *
+ *
+ * Caching feature:
+ *   - A cache index should be maintained which is faster than normal message request
+ *   - This should allow fast retrieval of message based on grp id or msg id
+ *
+ * Identity Exchange Service:
+ *   - As this is the point where data is accessed by both the GNP and GXS the identities
+ *     used to decrypt, encrypt and verify is handle here.
+ *
+ * Please note all function are blocking.
+ */
+class RsGeneralDataService
+{
+
+public:
+
+    RsGeneralDataService(){}
+    virtual ~RsGeneralDataService(){return;}
+
+    /*!
+     * Retrieves all msgs
+     * @param reqIds requested msg ids (grpId,msgId), leave msg list empty to get all msgs for the grp
+     * @param msg result of msg retrieval
+     * @param cache whether to store results of this retrieval in memory for faster later retrieval
+     * @return error code
+     */
+    virtual int retrieveNxsMsgs(const GxsMsgReq& reqIds, GxsMsgResult& msg, bool cache) = 0;
+
+    /*!
+     * Retrieves all groups stored
+     * @param grp retrieved groups
+     * @param withMeta this initialises the meta handles nxs grps
+     * @param cache whether to store retrieval in mem for faster later retrieval
+     * @return error code
+     */
+    virtual int retrieveNxsGrps(std::map<RsGxsGroupId, RsNxsGrp*>& grp, bool withMeta, bool cache) = 0;
+
+    /*!
+     * Retrieves meta data of all groups stored (most current versions only)
+     *
+     * @param grp if null grpIds entries are made, only meta for those grpId are retrieved \n
+     *            , if grpId is failed to be retrieved it will be erased from map
+     * @return error code
+     */
+    virtual int retrieveGxsGrpMetaData(std::map<RsGxsGroupId, RsGxsGrpMetaData*>& grp) = 0;
+
+    /*!
+     * Retrieves meta data of all groups stored (most current versions only)
+     * @param grpIds grpIds for which to retrieve meta data
+     * @param msgMeta meta data result as map of grpIds to array of metadata for that grpId
+     * @param cache whether to store retrieval in mem for faster later retrieval
+     * @return error code
+     */
+    virtual int retrieveGxsMsgMetaData(const std::vector<std::string>& grpIds, GxsMsgMetaResult& msgMeta) = 0;
+
+    /*!
+     * remove msgs in data store listed in msgIds param
+     * @param msgIds ids of messages to be removed
+     * @return error code
+     */
+    virtual int removeMsgs(const GxsMsgReq& msgIds) = 0;
+
+    /*!
+     * remove groups in data store listed in grpIds param
+     * @param grpIds ids of groups to be removed
+     * @return error code
+     */
+    virtual int removeGroups(const std::vector<RsGxsGroupId>& grpIds) = 0;
+
+    /*!
+     * @return the cache size set for this RsGeneralDataService in bytes
+     */
+    virtual uint32_t cacheSize() const = 0;
+
+    /*!
+     * @param size size of cache to set in bytes
+     */
+    virtual int setCacheSize(uint32_t size) = 0;
+
+    /*!
+     * Stores a list of signed messages into data store
+     * @param msg map of message and decoded meta data information
+     * @return error code
+     */
+    virtual int storeMessage(std::map<RsNxsMsg*, RsGxsMsgMetaData*>& msg) = 0;
+
+    /*!
+     * Stores a list of groups in data store
+     * @param grp map of group and decoded meta data
+     * @return error code
+     */
+    virtual int storeGroup(std::map<RsNxsGrp*, RsGxsGrpMetaData*>& grp) = 0;
+
+
+    /*!
+     * @param metaData
+     */
+    virtual int updateMessageMetaData(MsgLocMetaData* metaData) = 0;
+
+    /*!
+     * @param metaData
+     */
+    virtual int updateGroupMetaData(GrpLocMetaData* meta) = 0;
+
+
+    /*!
+     * Completely clear out data stored in
+     * and returns this to a state
+     * as it was when first constructed
+     */
+    virtual int resetDataStore() = 0;
+
+};
+
+
+
+
+#endif // RSGDS_H

libretroshare/src/gxs/rsgenexchange.cc

@@ -0,0 +1,289 @@
+#include "rsgenexchange.h"
+
+RsGenExchange::RsGenExchange(RsGeneralDataService *gds,
+                             RsNetworkExchangeService *ns, RsSerialType *serviceSerialiser, uint16_t servType)
+: mGenMtx("GenExchange"), mDataStore(gds), mNetService(ns), mSerialiser(serviceSerialiser), mServType(servType)
+{
+
+    mDataAccess = new RsGxsDataAccess(gds);
+
+}
+
+RsGenExchange::~RsGenExchange()
+{
+    // need to destruct in a certain order (prob a bad thing!)
+    delete mNetService;
+
+    delete mDataAccess;
+    mDataAccess = NULL;
+
+    delete mDataStore;
+    mDataStore = NULL;
+
+}
+
+
+void RsGenExchange::tick()
+{
+	mDataAccess->processRequests();
+
+	publishGrps();
+
+	publishMsgs();
+
+}
+
+
+bool RsGenExchange::getGroupList(const uint32_t &token, std::list<RsGxsGroupId> &groupIds)
+{
+	return mDataAccess->getGroupList(token, groupIds);
+
+}
+
+bool RsGenExchange::getMsgList(const uint32_t &token,
+                               GxsMsgIdResult &msgIds)
+{
+	return mDataAccess->getMsgList(token, msgIds);
+}
+
+bool RsGenExchange::getGroupMeta(const uint32_t &token, std::list<RsGroupMetaData> &groupInfo)
+{
+	std::list<RsGxsGrpMetaData*> metaL;
+	bool ok = mDataAccess->getGroupSummary(token, metaL);
+
+	std::list<RsGxsGrpMetaData*>::iterator lit = metaL.begin();
+
+	for(; lit != metaL.end(); lit++)
+	{
+		RsGroupMetaData m = *(*lit);
+		groupInfo.push_back(m);
+	}
+
+	std::list<RsGxsGrpMetaData*>::iterator cit = metaL;
+	for(; cit != metaL.end(); cit++)
+		delete *cit;
+
+	return ok;
+}
+
+
+bool RsGenExchange::getMsgMeta(const uint32_t &token,
+                               GxsMsgMetaMap &msgInfo)
+{
+	std::list<RsGxsMsgMetaData*> metaL;
+	GxsMsgMetaResult result;
+	bool ok = mDataAccess->getMsgSummary(token, result);
+
+	GxsMsgMetaResult::iterator mit = result.begin();
+
+	for(; mit != result.end(); mit++)
+	{
+		std::vector<RsGxsMsgMetaData*>& metaV = mit->second;
+		msgInfo[mit->first] = metaV;
+
+		std::vector<RsGxsMsgMetaData*>::iterator vit = metaV.begin();
+
+		for(; vit != metaV.end(); vit++)
+		{
+			delete *vit;
+		}
+	}
+
+	return ok;
+}
+
+bool RsGenExchange::getGroupData(const uint32_t &token, std::vector<RsGxsGrpItem *> grpItem)
+{
+
+	std::list<RsNxsGrp*> nxsGrps;
+	bool ok = mDataAccess->getGroupData(token, nxsGrps);
+
+	std::list<RsNxsGrp*>::iterator lit = nxsGrps.begin();
+
+	if(ok)
+	{
+		for(; lit != nxsGrps.end(); lit++)
+		{
+			RsTlvBinaryData& data = (*lit)->grp;
+			RsItem* item = mSerialiser->deserialise(data.bin_data, &data.bin_len);
+			RsGxsGrpItem* gItem = dynamic_cast<RsGxsGrpItem*>(item);
+			grpItem.push_back(gItem);
+			delete *lit;
+		}
+	}
+    return ok;
+}
+
+bool RsGenExchange::getMsgData(const uint32_t &token,
+                               GxsMsgDataMap &msgItems)
+{
+	NxsMsgDataResult msgResult;
+	bool ok = mDataAccess->getMsgData(token, msgResult);
+	NxsMsgDataResult::iterator mit = msgResult.begin();
+
+	if(ok)
+	{
+		for(; mit != msgResult.end(); mit++)
+		{
+			std::vector<RsGxsMsgItem*> gxsMsgItems;
+			const RsGxsGroupId& grpId = mit->first;
+			std::vector<RsNxsMsg*>& nxsMsgsV = mit->second;
+			std::vector<RsNxsMsg*>::iterator vit
+			= nxsMsgsV.begin();
+			for(; vit != nxsMsgsV.end(); vit++)
+			{
+				RsNxsMsg*& msg = *vit;
+
+				RsItem* item = mSerialiser->deserialise(msg->msg.bin_data,
+						&msg->msg.bin_len);
+				RsGxsMsgItem* mItem = dynamic_cast<RsGxsMsgItem*>(item);
+				gxsMsgItems.push_back(mItem);
+				delete msg;
+			}
+			msgItems[grpId] = gxsMsgItems;
+		}
+	}
+    return ok;
+}
+
+
+RsTokenService* RsGenExchange::getTokenService()
+{
+    return mDataAccess;
+}
+
+
+void RsGenExchange::notifyNewGroups(std::vector<RsNxsGrp *> &groups)
+{
+    std::vector<RsNxsGrp*>::iterator vit = groups.begin();
+
+    // store these for tick() to pick them up
+    for(; vit != groups.end(); vit++)
+       mReceivedGrps.push_back(*vit);
+
+}
+
+void RsGenExchange::notifyNewMessages(std::vector<RsNxsMsg *> messages)
+{
+    std::vector<RsNxsMsg*>::iterator vit = messages.begin();
+
+    // store these for tick() to pick them up
+    for(; vit != messages.end(); vit++)
+        mReceivedMsgs.push_back(*vit);
+
+}
+
+
+bool RsGenExchange::publishGroup(RsGxsGrpItem *grpItem)
+{
+
+	RsStackMutex stack(mGenMtx);
+
+	mGrpsToPublish.push_back(grpItem);
+
+    return true;
+}
+
+bool RsGenExchange::publishMsg(RsGxsMsgItem *msgItem)
+{
+
+	RsStackMutex stack(mGenMtx);
+
+	mMsgsToPublish.push_back(msgItem);
+    return true;
+}
+
+
+void RsGenExchange::publishMsgs()
+{
+	RsStackMutex stack(mGenMtx);
+
+	std::vector<RsGxsMsgItem*>::iterator vit = mMsgsToPublish.begin();
+
+	for(; vit != mMsgsToPublish.end(); )
+	{
+
+		RsNxsMsg* msg = new RsNxsMsg(mServType);
+		RsGxsMsgItem* msgItem = *vit;
+		uint32_t size = mSerialiser->size(msgItem);
+		char mData[size];
+		bool ok = mSerialiser->serialise(msgItem, mData, &size);
+
+		if(ok)
+		{
+			msg->metaData = new RsGxsMsgMetaData();
+			ok = mDataAccess->addMsgData(msg);
+
+			if(ok)
+			{
+				RsGxsMsgChange* mc = new RsGxsMsgChange();
+				mNotifications.push_back(mc);
+			}
+		}
+
+		if(!ok)
+		{
+#ifdef GEN_EXCH_DEBUG
+			std::cerr << "RsGenExchange::publishMsgs() failed to publish msg " << std::endl;
+#endif
+			delete msg;
+			continue;
+
+		}
+
+		delete msgItem;
+	}
+}
+
+void RsGenExchange::publishGrps()
+{
+
+	RsStackMutex stack(mGenMtx);
+
+	std::vector<RsGxsGrpItem*>::iterator vit = mGrpsToPublish.begin();
+
+	for(; vit != mGrpsToPublish.end();)
+	{
+
+		RsNxsGrp* grp = new RsNxsGrp(mServType);
+		RsGxsGrpItem* grpItem = *vit;
+		uint32_t size = mSerialiser->size(grpItem);
+
+		char gData[size];
+		bool ok = mSerialiser->serialise(grpItem, gData, &size);
+
+		if(ok)
+		{
+			grp->metaData = new RsGxsGrpMetaData();
+			ok = mDataAccess->addGroupData(grp);
+			RsGxsGroupChange* gc = RsGxsGroupChange();
+			mNotifications.push_back(gc);
+		}
+
+		if(!ok)
+		{
+
+#ifdef GEN_EXCH_DEBUG
+			std::cerr << "RsGenExchange::publishGrps() failed to publish grp " << std::endl;
+#endif
+			delete grp;
+			continue;
+		}
+
+		delete grpItem;
+
+		vit = mGrpsToPublish.erase(vit);
+	}
+
+}
+void RsGenExchange::processRecvdData() {
+}
+
+
+void RsGenExchange::processRecvdMessages() {
+}
+
+
+void RsGenExchange::processRecvdGroups() {
+}
+

libretroshare/src/gxs/rsgenexchange.h

@@ -0,0 +1,231 @@
+#ifndef RSGENEXCHANGE_H
+#define RSGENEXCHANGE_H
+
+/*
+ * libretroshare/src/retroshare: rsphoto.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Christopher Evi-Parker, Robert Fernie
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <queue>
+
+#include "rsgxs.h"
+#include "rsgds.h"
+#include "rsnxs.h"
+#include "rsgxsdataaccess.h"
+#include "retroshare/rsgxsservice.h"
+#include "serialiser/rsnxsitems.h"
+
+typedef std::map<RsGxsGroupId, std::vector<RsGxsMsgItem*> > GxsMsgDataMap;
+typedef std::map<RsGxsGroupId, RsGxsGrpItem*> GxsGroupDataMap;
+typedef std::map<RsGxsGroupId, std::vector<RsMsgMetaData> > GxsMsgMetaMap;
+
+/*!
+ * This should form the parent class to \n
+ * all gxs services. This provides access to service's msg/grp data \n
+ * management/publishing/sync features
+ *
+ * Features: \n
+ *         a. Data Access:
+ *              Provided by handle to RsTokenService. This ensure consistency
+ *              of requests and hiearchy of groups -> then messages which are
+ *              sectioned by group ids.
+ *              The one caveat is that redemption of tokens are done through
+ *              the backend of this class
+ *         b. Publishing:
+ *              Methods are provided to publish msg and group items and also make
+ *              changes to meta information of both item types
+ *         c. Sync/Notification:
+ *              Also notifications are made here on receipt of new data from
+ *              connected peers
+ */
+class RsGenExchange : public RsGxsService
+{
+public:
+
+    /*!
+     * Constructs a RsGenExchange object, the owner ship of gds, ns, and serviceserialiser passes
+     * onto the constructed object
+     * @param gds Data service needed to act as store of message
+     * @param ns Network service needed to synchronise data with rs peers
+     * @param serviceSerialiser The users service needs this \n
+     *        in order for gen exchange to deal with its data types
+     */
+    RsGenExchange(RsGeneralDataService* gds, RsNetworkExchangeService* ns, RsSerialType* serviceSerialiser, uint16_t mServType);
+
+    virtual ~RsGenExchange();
+
+
+    /** S: Observer implementation **/
+
+    /*!
+     * @param messages messages are deleted after function returns
+     */
+    void notifyNewMessages(std::vector<RsNxsMsg*> messages);
+
+    /*!
+     * @param messages messages are deleted after function returns
+     */
+    void notifyNewGroups(std::vector<RsNxsGrp*>& groups);
+
+    /** E: Observer implementation **/
+
+    /*!
+     * This is called by Gxs service runner
+     * periodically, use to implement non
+     * blocking calls
+     */
+    void tick();
+
+    /*!
+     *
+     * @return handle to token service handle for making
+     * request to this gxs service
+     */
+    RsTokenService* getTokenService();
+
+protected:
+
+    /** data access functions **/
+
+    /*!
+     * Retrieve group list for a given token
+     * @param token
+     * @param groupIds
+     * @return false if token cannot be redeemed, if false you may have tried to redeem when not ready
+     */
+    bool getGroupList(const uint32_t &token, std::list<RsGxsGroupId> &groupIds);
+
+    /*!
+     * Retrieve msg list for a given token sectioned by group Ids
+     * @param token token to be redeemed
+     * @param msgIds a map of grpId -> msgList (vector)
+     */
+    bool getMsgList(const uint32_t &token, GxsMsgIdResult &msgIds);
+
+
+    /*!
+     * retrieve group meta data associated to a request token
+     * @param token
+     * @param groupInfo
+     */
+    bool getGroupMeta(const uint32_t &token, std::list<RsGroupMetaData> &groupInfo);
+
+    /*!
+     * retrieves message meta data associated to a request token
+     * @param token token to be redeemed
+     * @param msgInfo the meta data to be retrieved for token store here
+     */
+    bool getMsgMeta(const uint32_t &token, GxsMsgMetaMap &msgInfo);
+
+    /*!
+     * retrieves group data associated to a request token
+     * @param token token to be redeemed for grpitem retrieval
+     * @param grpItem the items to be retrieved for token are stored here
+     */
+    bool getGroupData(const uint32_t &token, std::vector<RsGxsGrpItem*> grpItem);
+
+    /*!
+     * retrieves message data associated to a request token
+     * @param token token to be redeemed for message item retrieval
+     * @param msgItems
+     */
+    bool getMsgData(const uint32_t &token, GxsMsgDataMap& msgItems);
+
+protected:
+
+    /** Modifications **/
+
+    /*!
+     * Enables publication of a group item
+     * If the item exists already this is simply versioned
+     * This will induce a related change message
+     * Ownership of item passes to this rsgenexchange
+     * @param grpItem
+     * @param
+     */
+    bool publishGroup(RsGxsGrpItem* grpItem);
+
+    /*!
+     * Enables publication of a message item
+     * If the item exists already this is simply versioned
+     * This will induce a related a change message
+     * Ownership of item passes to this rsgenexchange
+     * @param msgItem
+     * @return false if msg creation failed.
+     */
+    bool publishMsg(RsGxsMsgItem* msgItem);
+
+
+protected:
+
+    /** Notifications **/
+
+    /*!
+     * This confirms this class as an abstract one that \n
+     * should not be instantiated \n
+     * The deriving class should implement this function \n
+     * as it is called by the backend GXS system to \n
+     * update client of changes which should \n
+     * instigate client to retrieve new content from the system
+     * @param changes the changes that have occured to data held by this service
+     */
+    virtual void notifyChanges(std::vector<RsGxsNotify*>& changes) = 0;
+
+public:
+
+    void processRecvdData();
+
+    void processRecvdMessages();
+
+    void processRecvdGroups();
+
+    void publishGrps();
+
+    void publishMsgs();
+
+private:
+
+    RsMutex mGenMtx;
+    RsGxsDataAccess* mDataAccess;
+    RsGeneralDataService* mDataStore;
+    RsNetworkExchangeService *mNetService;
+    RsSerialType *mSerialiser;
+
+    std::vector<RsNxsMsg*> mReceivedMsgs;
+    std::vector<RsNxsGrp*> mReceivedGrps;
+
+    std::vector<RsGxsGrpItem*> mGrpsToPublish;
+    std::vector<RsGxsMsgItem*> mMsgsToPublish;
+
+    std::vector<RsGxsNotify*> mNotifications;
+
+    /// service type
+    uint16_t mServType;
+
+
+private:
+
+    std::vector<RsGxsChange*> mChanges;
+};
+
+#endif // RSGENEXCHANGE_H

libretroshare/src/gxs/rsgixs.h

@@ -0,0 +1,231 @@
+#ifndef RSGIXS_H
+#define RSGIXS_H
+
+/*
+ * libretroshare/src/gxs: gxs.h
+ *
+ * General Identity Exchange Service interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie, Christopher Evi-Prker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "gxs/rsgxs.h"
+
+#include <openssl/ssl.h>
+#include <set>
+
+/*!
+ * GIXP: General Identity Exchange Service.
+ *
+ * As we're always running into troubles with GPG signatures... we are going to
+ * create a layer of RSA Keys for the following properties:
+ *
+ * 1) RSA Keys can be Anonymous, Self-Signed with Pseudonym, Signed by GPG Key.
+ * To clarify:
+ *    a. This forms a layer of keys stay between GPG and pub/priv publish key ?
+ *    b. Difference between anonymous and pseudonym keys?
+ *            - Anonymous cannot be signed?
+ *            -
+ *    c. To some extent this determines security model of RsGeneralExchangeService
+
+ *	- Anonymous & Pseudonym Keys will be shared network-wide (Hop by Hop).
+        - GPG signed Keys will only be shared if we can validate the signature
+                (providing similar behaviour to existing GPG Keys).
+        - GPG signed Keys can optionally be marked for Network-wide sharing.
+ * 2) These keys can be used anywhere, specifically in the protocols described below.
+ * 3) These keys can be used to sign, encrypt, verify & decrypt
+ * 4) Keys will never need to be directly accessed - stored in this class.
+ *     a. I guess can work solely through Id
+ *     b. Use Case: Receivve a message, has a key id, request
+ * 5) They will be cached locally and exchanged p2p, by pull request.
+ * 6) This class will use the generalised packet storage for efficient caching & loading.
+ * 7) Data will be stored encrypted.
+ */
+
+
+/*!
+ * Storage class for private and public publish keys
+ *
+ */
+class GixsKey
+{
+        KeyRef mKeyId;
+
+        /// public key
+        EVP_PKEY *mPubKey;
+
+        ///  NULL if non-existant */
+        EVP_PKEY *mPrivKey;
+};
+
+/*!
+ *
+ *
+ */
+class KeyRef {
+
+    std::string refId;
+
+};
+
+
+class KeyRefSet {
+    std::set<KeyRef> mKeyRefSet;
+};
+
+class SignatureSet {
+    std::set<RsGxsSignature> mSignatureSet;
+};
+
+/*!
+ *
+ *
+ */
+class RsGxsSignature {
+
+    KeyRef mKeyRef;
+};
+
+/*!
+ * This is the actual identity \n
+ * In a sense the group description with the GixsKey the "message"
+ */
+class RsGixsProfile {
+
+public:
+
+    KeyRef mKeyRef;
+    std::string name;
+
+    /// may be superseded by newer timestamps
+    time_t mTimeStamp;
+    uint32_t mProfileType;
+
+    // TODO: add permissions members
+
+    RsGxsSignature mSignature;
+
+};
+
+/*!
+ * Retroshare general identity exchange service
+ *
+ * Purpose: \n
+ *   Provides a means to distribute identities among peers \n
+ *   Also provides encyption, decryption, verification, \n
+ *   and signing functionality using any created or received identities \n
+ *
+ * This may best be implemented as a singleton like current AuthGPG? \n
+ *
+ */
+class RsIdentityExchangeService : RsGxsService
+{
+public:
+
+    enum IdentityType { Pseudonym, Signed, Anonymous };
+
+    RsGixs();
+
+    /*!
+     * creates gixs profile and shares it
+     * @param profile
+     * @param type the type of profile to create, self signed, anonymous, and GPG signed
+     */
+    virtual bool createKey(RsGixsProfile& profile, uint32_t type) = 0; /* fills in mKeyId, and signature */
+
+    /*!
+     * Use to query a whether given key is available by its key reference
+     * @param keyref the keyref of key that is being checked for
+     * @return true if available, false otherwise
+     */
+    virtual bool haveKey(const KeyRef& keyref) = 0;
+
+    /*!
+     * Use to query whether private key member of the given key reference is available
+     * @param keyref the KeyRef of the key being checked for
+     * @return true if private key is held here, false otherwise
+     */
+    virtual bool havePrivateKey(const KeyRef& keyref) = 0;
+
+    /*!
+     * Use to request a given key reference
+     * @param keyref the KeyRef of the key being requested
+     * @return will
+     */
+    virtual bool requestKey(const KeyRef& keyref) = 0;
+
+    /*!
+     * Retrieves a key identity
+     * @param keyref
+     * @return a pointer to a valid profile if successful, otherwise NULL
+     *
+     */
+    virtual RsGixsProfile* getProfile(const KeyRef& keyref) = 0;
+
+
+    /*** process data ***/
+
+    /*!
+     * Use to sign data with a given key
+     * @param keyref the key to sign the data with
+     * @param data the data to be signed
+     * @param dataLen the length of the data
+     * @param signature is set with the signature from signing with keyref
+     * @return false if signing failed, true otherwise
+     */
+    virtual bool sign(const KeyRef& keyref, unsigned char* data, uint32_t dataLen, std::string& signature) = 0;
+
+    /*!
+     * Verify that the data is signed by the key owner
+     * @param keyref
+     * @param data
+     * @param dataLen
+     * @param signature
+     * @return false if verification failed, false otherwise
+     */
+    virtual bool verify(const KeyRef& keyref, unsigned char* data, int dataLen, std::string& signature) = 0;
+
+    /*!
+     * Attempt to decrypt data with a given key
+     * @param keyref
+     * @param data data to be decrypted
+     * @param dataLen length of data
+     * @param decryptedData decrypted data
+     * @param decryptDataLen length of decrypted data
+     * @return false
+     */
+    virtual bool decrypt(const KeyRef& keyref, unsigned char* data, int dataLen,
+                 unsigned char*& decryptedData, uint32_t& decyptDataLen) = 0;
+
+    /*!
+     * Attempt to encrypt data with a given key
+     * @param keyref
+     * @param data data to be encrypted
+     * @param dataLen length of data
+     * @param encryptedData encrypted data
+     * @param encryptDataLen length of encrypted data
+     */
+    virtual bool encrypt(const KeyRef& keyref, unsigned char* data, int dataLen,
+                 unsigned char*& encryptedData, uint32_t& encryptDataLen) = 0;
+
+};
+
+
+#endif // RSGIXS_H

libretroshare/src/gxs/rsgroups.h

@@ -0,0 +1,151 @@
+//
+// This is a first attempt at managing groups and group permissions
+// in libretroshare.
+//
+// Rules:
+// 	1 - Groups are handled GPG-wise, and not SSLId wise, but we need to provide functions
+// 		for SSLIds since that's what peers are represented by.
+//
+// 	2 - A peer can be in different groups. That will be useful for e.g. file sharing.
+//
+// 	3 - Global permissions always prevail over permissions given at the group level. For 
+// 	   instance for discovery between two peers that belong to groups G1 and G2:
+//
+//                             | Global discovery flag ON | Global disc flag OFF |
+//         --------------------+--------------------------+----------------------+
+//         Group disc(G1,G2)=1 |           Allow          |     Don't allow      |
+//         Group disc(G1,G2)=0 |         Don't allow      |     Don't allow      |
+//
+//    4 - Binary group permissions are OR-ed between pairs of groups. If peers (A,B) can be matched to
+//    	two different pairs of groups, the flags are combined with an OR.
+//
+// libretroshare side
+// ==================
+//   We need a matrix for binary group flags, i.e. flags for pair of groups, for:
+//
+//      Turtle traffic: 
+//      	* does the turtle traffic can go from peers of a group to another
+//      	* default: yes
+//
+//      Discovery:
+//      	* does the turtle traffic can go from peers of a group to another
+//      	* default: yes. Overriden by discovery status in config.
+//
+//      Public chat lobby advertisement:
+//      	* public lobby coming from peers of a group go to peers of another group
+//      	* default: yes
+//
+//      Relay connexions:
+//       * allow relay connexions between people of group A and group B.
+//
+//      General Exchange Services:
+//       * Allow data exchanged from one peers of one group to peers of another group
+//
+//   We need per-group permission flags for:
+//
+//      File download permission: 
+//       * each shared directory belongs to a list of groups
+//      	* each shared directory has a set of DL flags (browsable B,network wide N) for groups and for global.
+//      	* files are listed with unix-like permissions style: 
+//      			---- : file can't be transfered at all. 
+//      			bn-- : file is browsable and turtle-able by friends of its groups only.
+//      			b-b- : file is browsable by everybody
+//      			--b- : file is browsable by everybody that does not belong to its groups (can be useful to isolate people in a group)
+//      			-n-- : file is turtle-able by people that belong to its groups only.
+//      			bnbn : file is always accessible in any ways. That's what file sharing is about. Should be the default ;-)
+//
+//      	We need to propose some pre-initialized settings to the user, so that he won't have to bother with flags, but we
+//      	need to allow precisely tweaking them as well:
+//      		Directory is 
+//      			- Accessible for this group only (allow select a group)
+//      			- Fully shared anonymously
+//      			- [...]
+//
+//      	If the shared directory has no groups, than the first two flags are meaningless.
+//  
+// GUI side
+// ========
+// 	- we need a tab in config to manage groups and group flag permission matrix
+// 	- when sharing a directory, the user can act on permissions: setting which groups a 
+// 	  directory belongs to.
+//
+// Classes for groups. To be displatched in rstypes.h ?
+//
+
+#pragma once
+
+typedef std::string SSLId ;
+typedef std::string GPGId ;
+typedef uint64_t    RsGroupId ;
+
+class RsPeerGroupInfo
+{
+	public:
+		RsGroupId group_id ;				// Id of the group, should be random and unique in the RS session.
+		std::string group_name ;		// user-defined name of the group ("parents", "friends", etc)
+		std::set<GPGId> elements ;		// people in the group. GPG-wise.
+};
+
+class GroupFlagsMatrix
+{
+	public:
+		// flags
+
+		static const uint32_t GROUP_MATRIX_FLAG_DISCOVERY    = 0x1 ;
+		static const uint32_t GROUP_MATRIX_FLAG_TURTLE       = 0x2 ;
+		static const uint32_t GROUP_MATRIX_FLAG_PUBLIC_LOBBY = 0x4 ;
+		static const uint32_t GROUP_MATRIX_FLAG_RELAY        = 0x8 ;
+
+		// returns/set all flags for a given pair of groups. Used by the 
+		// GUI to draw/modify group flags.
+		//
+		uint32_t getGroupFlags(RsGroupId grp1,RsGroupId grp2) const ;	
+		void setGroupFlags(uint32_t flags,RsGroupId grp1,RsGroupId grp2) const ;	
+
+		// Returns a given flag for two people in particular. Used by services to 
+		// know what to do or not do.
+		//
+		bool groupPermission(uint32_t group_flag,GPGId peer_1,GPGId peer2) const ;
+
+	private:
+		std::vector<uint32_t> _flags ;	// vector of size number_of_groups^2
+		std::map<RsGroupId,int> _group_entries ; // index to read the flags vector.
+};
+
+// Interface class for Groups. Should go in libretroshare/stc/retroshare/rsgroups.h
+//
+
+class RsGroupManagement
+{
+	public:
+		// Group handling
+
+		// Returns false is the group does not exist. Otherwise, fills the GroupInfo struct.
+		//
+		virtual bool getGroupInfo(const RsGroupId& id,RsPeerGroupInfo& info) = 0 ;
+
+		// Adds a friend to a group.
+		virtual bool getGroupInfo(const RsGroupId& id,RsPeerGroupInfo& info) = 0 ;
+
+		// Group permission handling
+		//
+		virtual bool allow_TwoPeers_TurtleTraffic     (const SSLId& p1,const SSLId& p2) = 0 ;
+		virtual bool allow_TwoPeers_Discovery         (const SSLId& p1,const SSLId& p2) = 0 ;
+		virtual bool allow_TwoPeers_LobbyAdvertisement(const SSLId& p1,const SSLId& p2) = 0 ;
+                virtual bool allow_TwoPeers_GeneralExchangeService(const SSLId& p1,const SSLId& p2) = 0 ;
+		// [...]
+};
+
+class p3GroupManagement: public RsGroupManagement, public p3Config
+{
+	public:
+		// [...]
+		
+	private:
+		std::map<RsPeerGroupInfo> _groups ;
+};
+
+// This is the entry point for external interface.
+//
+extern RsGroupManagement *rsGroups ;
+

libretroshare/src/gxs/rsgxs.h

@@ -0,0 +1,57 @@
+#ifndef RSGXS_H
+#define RSGXS_H
+
+/*
+ * libretroshare/src/gxs   : rsgxs.h
+ *
+ * GXS  interface for RetroShare.
+ * Convenience header
+ *
+ * Copyright 2011 Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ * This is *THE* auth manager. It provides the web-of-trust via
+ * gpgme, and authenticates the certificates that are managed
+ * by the sublayer AuthSSL.
+ *
+ */
+
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+#include <set>
+#include <map>
+
+#include "rsnxsobserver.h"
+
+
+
+class RsGxsService : public RsNxsObserver
+{
+
+public:
+
+    RsGxsService();
+    virtual ~RsGxsService();
+
+    virtual void tick() = 0;
+
+};
+
+#endif // RSGXS_H

libretroshare/src/gxs/rsgxsdata.cc

@@ -0,0 +1,252 @@
+
+#include "rsgxsdata.h"
+#include "serialiser/rsbaseserial.h"
+
+RsGxsGrpMetaData::RsGxsGrpMetaData()
+{
+
+}
+
+uint32_t RsGxsGrpMetaData::serial_size()
+{
+    uint32_t s = 8; // header size
+
+    s += GetTlvStringSize(mGroupId);
+    s += GetTlvStringSize(mOrigGrpId);
+    s += GetTlvStringSize(mGroupName);
+    s += 4;
+    s += 4;
+    s += GetTlvStringSize(mAuthorId);
+    s += adminSign.TlvSize();
+    s += keys.TlvSize();
+    s += idSign.TlvSize();
+
+    return s;
+}
+
+void RsGxsGrpMetaData::clear(){
+
+    mGroupId.clear();
+    mOrigGrpId.clear();
+    mAuthorId.clear();
+    mGroupName.clear();
+    mPublishTs = 0;
+    mGroupFlags = 0;
+    mPop = 0;
+    mMsgCount = 0;
+    mGroupStatus = 0;
+    mLastPost = 0;
+    mSubscribeFlags = 0;
+
+    adminSign.TlvClear();
+    keys.TlvClear();
+    idSign.TlvClear();
+}
+
+bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize)
+{
+
+    uint32_t tlvsize = serial_size() ;
+    uint32_t offset = 0;
+
+    if (pktsize < tlvsize)
+            return false; /* not enough space */
+
+    pktsize = tlvsize;
+
+    bool ok = true;
+
+    ok &= setRsItemHeader(data, tlvsize, 0, tlvsize);
+
+#ifdef GXS_DEBUG
+    std::cerr << "RsGxsGrpMetaData serialise()" << std::endl;
+    std::cerr << "RsGxsGrpMetaData serialise(): Header: " << ok << std::endl;
+    std::cerr << "RsGxsGrpMetaData serialise(): Size: " << tlvsize << std::endl;
+#endif
+
+    /* skip header */
+    offset += 8;
+
+    ok &= SetTlvString(data, tlvsize, &offset, 0, mGroupId);
+    ok &= SetTlvString(data, tlvsize, &offset, 0, mOrigGrpId);
+    ok &= SetTlvString(data, tlvsize, &offset, 0, mGroupName);
+    ok &= setRawUInt32(data, tlvsize, &offset, mGroupFlags);
+    ok &= setRawUInt32(data, tlvsize, &offset, mPublishTs);
+    ok &= SetTlvString(data, tlvsize, &offset, 0, mAuthorId);
+
+    ok &= adminSign.SetTlv(data, tlvsize, &offset);
+    ok &= keys.SetTlv(data, tlvsize, &offset);
+    ok &= idSign.SetTlv(data, tlvsize, &offset);
+
+    return ok;
+}
+
+bool RsGxsGrpMetaData::deserialise(void *data, uint32_t &pktsize)
+{
+
+    uint32_t offset = 8; // skip the header
+    uint32_t rssize = getRsItemSize(data);
+
+    bool ok = true ;
+
+    ok &= rssize == pktsize;
+
+    if(!ok) return false;
+
+    ok &= GetTlvString(data, pktsize, &offset, 0, mGroupId);
+    ok &= GetTlvString(data, pktsize, &offset, 0, mOrigGrpId);
+    ok &= GetTlvString(data, pktsize, &offset, 0, mGroupName);
+    ok &= getRawUInt32(data, pktsize, &offset, &mGroupFlags);
+    ok &= getRawUInt32(data, pktsize, &offset, &mPublishTs);
+    ok &= GetTlvString(data, pktsize, &offset, 0, mAuthorId);
+
+    ok &= adminSign.GetTlv(data, pktsize, &offset);
+    ok &= keys.GetTlv(data, pktsize, &offset);
+    ok &= idSign.GetTlv(data, pktsize, &offset);
+
+    return ok;
+}
+
+RsGxsMsgMetaData::RsGxsMsgMetaData(){
+
+}
+
+uint32_t RsGxsMsgMetaData::serial_size()
+{
+
+    uint32_t s = 8; // header size
+
+    s += GetTlvStringSize(mGroupId);
+    s += GetTlvStringSize(mMsgId);
+    s += GetTlvStringSize(mThreadId);
+    s += GetTlvStringSize(mParentId);
+    s += GetTlvStringSize(mOrigMsgId);
+    s += GetTlvStringSize(mAuthorId);
+
+    s += pubSign.TlvSize();
+    s += idSign.TlvSize();
+    s += GetTlvStringSize(mMsgName);
+    s += 4;
+    s += 4;
+
+    return s;
+}
+
+void RsGxsMsgMetaData::clear()
+{
+    mGroupId.clear();
+    mMsgId.clear();
+    mThreadId.clear();
+    mParentId.clear();
+    mAuthorId.clear();
+    mOrigMsgId.clear();
+    mMsgName.clear();
+
+    pubSign.TlvClear();
+    idSign.TlvClear();
+
+    mPublishTs = 0;
+    mMsgFlags = 0;
+    mMsgStatus = 0;
+    mChildTs = 0;
+}
+
+bool RsGxsMsgMetaData::serialise(void *data, uint32_t *size)
+{
+    uint32_t tlvsize = serial_size() ;
+    uint32_t offset = 0;
+
+    if (*size < tlvsize)
+            return false; /* not enough space */
+
+    *size = tlvsize;
+
+    bool ok = true;
+
+    ok &= setRsItemHeader(data, tlvsize, 0, tlvsize);
+
+#ifdef GXS_DEBUG
+    std::cerr << "RsGxsGrpMetaData serialise()" << std::endl;
+    std::cerr << "RsGxsGrpMetaData serialise(): Header: " << ok << std::endl;
+    std::cerr << "RsGxsGrpMetaData serialise(): Size: " << tlvsize << std::endl;
+#endif
+
+    /* skip header */
+    offset += 8;
+
+    ok &= SetTlvString(data, *size, &offset, 0, mGroupId);
+    ok &= SetTlvString(data, *size, &offset, 0, mMsgId);
+    ok &= SetTlvString(data, *size, &offset, 0, mThreadId);
+    ok &= SetTlvString(data, *size, &offset, 0, mParentId);
+    ok &= SetTlvString(data, *size, &offset, 0, mOrigMsgId);
+    ok &= SetTlvString(data, *size, &offset, 0, mAuthorId);
+
+    ok &= pubSign.SetTlv(data, *size, &offset);
+    ok &= idSign.SetTlv(data, *size, &offset);
+    ok &= SetTlvString(data, *size, &offset, 0, mMsgName);
+    ok &= setRawUInt32(data, *size, &offset, mPublishTs);
+    ok &= setRawUInt32(data, *size, &offset, mMsgFlags);
+
+    return ok;
+}
+
+
+bool RsGxsMsgMetaData::deserialise(void *data, uint32_t *size)
+{
+    uint32_t offset = 8; // skip the header
+    uint32_t rssize = getRsItemSize(data);
+
+    bool ok = true ;
+
+    ok &= rssize == *size;
+
+    if(!ok) return false;
+
+    ok &= GetTlvString(data, *size, &offset, 0, mGroupId);
+    ok &= GetTlvString(data, *size, &offset, 0, mMsgId);
+    ok &= GetTlvString(data, *size, &offset, 0, mThreadId);
+    ok &= GetTlvString(data, *size, &offset, 0, mParentId);
+    ok &= GetTlvString(data, *size, &offset, 0, mOrigMsgId);
+    ok &= GetTlvString(data, *size, &offset, 0, mAuthorId);
+
+    ok &= pubSign.GetTlv(data, *size, &offset);
+    ok &= idSign.GetTlv(data, *size, &offset);
+    ok &= GetTlvString(data, *size, &offset, 0, mMsgName);
+    uint32_t t;
+    ok &= getRawUInt32(data, *size, &offset, &t);
+    mPublishTs = t;
+    ok &= getRawUInt32(data, *size, &offset, &mMsgFlags);
+
+    return ok;
+}
+
+void RsGxsGrpMetaData::operator =(const RsGroupMetaData& rMeta)
+{
+	this->mAuthorId = rMeta.mAuthorId;
+	this->mGroupFlags = rMeta.mGroupFlags;
+	this->mGroupId = rMeta.mGroupId;
+	this->mGroupStatus = rMeta.mGroupStatus ;
+	this->mLastPost = rMeta.mLastPost;
+	this->mMsgCount = rMeta.mMsgCount ;
+	this->mPop = rMeta.mPop;
+	this->mPublishTs = rMeta.mPublishTs;
+	this->mSubscribeFlags = rMeta.mSubscribeFlags;
+	this->mGroupName = rMeta.mGroupName;
+}
+
+void RsGxsMsgMetaData::operator =(const RsMsgMetaData& rMeta)
+{
+	this->mAuthorId = rMeta.mAuthorId;
+	this->mChildTs = rMeta.mChildTs ;
+	this->mGroupId = rMeta.mGroupId;
+	this->mMsgFlags = rMeta.mMsgFlags ;
+	this->mMsgId = rMeta.mMsgId ;
+	this->mMsgName = rMeta.mMsgName;
+	this->mMsgStatus = rMeta.mMsgStatus;
+	this->mOrigMsgId = rMeta.mOrigMsgId;
+	this->mParentId = rMeta.mParentId ;
+	this->mPublishTs = rMeta.mPublishTs ;
+	this->mThreadId = rMeta.mThreadId;
+}
+
+

libretroshare/src/gxs/rsgxsdata.h

@@ -0,0 +1,95 @@
+#ifndef RSGXSMETA_H
+#define RSGXSMETA_H
+
+#include <string>
+
+#include "serialiser/rsserial.h"
+#include "serialiser/rstlvbase.h"
+#include "serialiser/rstlvtypes.h"
+#include "serialiser/rstlvkeys.h"
+#include "serialiser/rsgxsitems.h"
+
+typedef std::string RsGxsGroupId;
+typedef std::string RsGxsMessageId;
+typedef std::pair<RsGxsGroupId, RsGxsMessageId> RsGxsGrpMsgIdPair;
+
+class RsGroupMetaData;
+class RsMsgMetaData;
+
+class RsGxsGrpMetaData
+{
+public:
+
+    RsGxsGrpMetaData();
+    bool deserialise(void *data, uint32_t &pktsize);
+    bool serialise(void* data, uint32_t &pktsize);
+    uint32_t serial_size();
+    void clear();
+    void operator =(const RsGroupMetaData& rMeta);
+
+    RsGxsGroupId mGroupId;
+    RsGxsGroupId mOrigGrpId;
+    std::string mGroupName;
+    uint32_t    mGroupFlags;
+    uint32_t    mPublishTs;
+    std::string mAuthorId;
+
+
+    RsTlvKeySignature adminSign;
+    RsTlvSecurityKeySet keys;
+    RsTlvKeySignature idSign;
+
+
+
+    // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.
+
+    uint32_t    mSubscribeFlags;
+
+    uint32_t    mPop; // HOW DO WE DO THIS NOW.
+    uint32_t    mMsgCount; // ???
+    time_t      mLastPost; // ???
+
+    uint32_t    mGroupStatus;
+
+};
+
+
+
+
+class RsGxsMsgMetaData
+{
+public:
+
+    RsGxsMsgMetaData();
+    bool deserialise(void *data, uint32_t *size);
+    bool serialise(void* data, uint32_t *size);
+    uint32_t serial_size();
+    void clear();
+    void operator =(const RsMsgMetaData& rMeta);
+
+    RsGxsGroupId mGroupId;
+    RsGxsMessageId mMsgId;
+
+    RsGxsMessageId mThreadId;
+    RsGxsMessageId mParentId;
+    RsGxsMessageId mOrigMsgId;
+    std::string mAuthorId;
+
+    RsTlvKeySignature pubSign;
+    RsTlvKeySignature idSign;
+
+    std::string mMsgName;
+    time_t      mPublishTs;
+    uint32_t    mMsgFlags; // Whats this for?
+
+    // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.
+    // normally READ / UNREAD flags. LOCAL Data.
+    uint32_t    mMsgStatus;
+    time_t      mChildTs;
+
+};
+
+
+
+
+#endif // RSGXSMETA_H

libretroshare/src/gxs/rsgxsdataaccess.cc

@@ -0,0 +1,923 @@
+#include "rsgxsdataaccess.h"
+
+/*
+ * libretroshare/src/retroshare: rsgxsdataaccess.cc
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Robert Fernie, Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+// This bit will be filled out over time.
+#define RS_TOKREQOPT_MSG_VERSIONS	0x0001		// MSGRELATED: Returns All MsgIds with OrigMsgId = MsgId.
+#define RS_TOKREQOPT_MSG_ORIGMSG	0x0002		// MSGLIST: All Unique OrigMsgIds in a Group.
+#define RS_TOKREQOPT_MSG_LATEST		0x0004		// MSGLIST: All Latest MsgIds in Group. MSGRELATED: Latest MsgIds for Input Msgs.
+
+#define RS_TOKREQOPT_MSG_THREAD		0x0010		// MSGRELATED: All Msgs in Thread. MSGLIST: All Unique Thread Ids in Group.
+#define RS_TOKREQOPT_MSG_PARENT		0x0020		// MSGRELATED: All Children Msgs.
+
+#define RS_TOKREQOPT_MSG_AUTHOR		0x0040		// MSGLIST: Messages from this AuthorId
+
+
+// Status Filtering... should it be a different Option Field.
+#define RS_TOKREQOPT_GROUP_UPDATED	0x0100		// GROUPLIST: Groups that have been updated.
+#define RS_TOKREQOPT_MSG_UPDATED	0x0200		// MSGLIST: Msg that have been updated from specified groups.
+#define RS_TOKREQOPT_MSG_UPDATED	0x0200		// MSGLIST: Msg that have been updated from specified groups.
+
+
+
+// Read Status.
+#define RS_TOKREQOPT_READ		0x0001
+#define RS_TOKREQOPT_UNREAD		0x0002
+
+#define RS_TOKREQ_ANSTYPE_LIST		0x0001
+#define RS_TOKREQ_ANSTYPE_SUMMARY	0x0002
+#define RS_TOKREQ_ANSTYPE_DATA		0x0003
+
+	const uint8_t RsGxsDataAccess::GXS_REQUEST_STATUS_FAILED = 0;
+	const uint8_t RsGxsDataAccess::GXS_REQUEST_STATUS_PENDING = 1;
+	const uint8_t RsGxsDataAccess::GXS_REQUEST_STATUS_PARTIAL = 2;
+	const uint8_t RsGxsDataAccess::GXS_REQUEST_STATUS_FINISHED_INCOMPLETE = 3;
+	const uint8_t RsGxsDataAccess::GXS_REQUEST_STATUS_COMPLETE = 4;
+	const uint8_t RsGxsDataAccess::GXS_REQUEST_STATUS_DONE = 5;			 // ONCE ALL DATA RETRIEVED.
+
+RsGxsDataAccess::RsGxsDataAccess(RsGeneralDataService* ds)
+ : mDataStore(ds)
+{
+}
+
+
+bool RsGxsDataAccess::requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts,
+		const std::list<std::string> &groupIds)
+{
+	GxsRequest* req = NULL;
+	uint32_t reqType = opts.mReqType;
+
+	if(reqType & GXS_REQUEST_TYPE_GROUP_META)
+	{
+		GroupMetaReq* gmr = new GroupMetaReq();
+		gmr->mGroupIds = groupIds;
+		req = gmr;
+	}
+	else if(reqType & GXS_REQUEST_TYPE_GROUP_DATA)
+	{
+		GroupDataReq* gdr = new GroupDataReq();
+		gdr->mGroupIds = groupIds;
+		req = gdr;
+	}
+	else if(reqType & GXS_REQUEST_TYPE_GROUP_IDS)
+	{
+		GroupIdReq* gir = new GroupIdReq();
+		gir->mGroupIds = groupIds;
+		req = gir;
+	}
+
+	if(req == NULL)
+	{
+		std::cerr << "RsGxsDataAccess::requestMsgInfo() request type not recognised, type "
+				  << reqType << std::endl;
+		return false;
+	}else
+	{
+		generateToken(token);
+		std::cerr << "RsGxsDataAccess::requestMsgInfo() gets Token: " << token << std::endl;
+	}
+
+	setReq(req, token, ansType, opts);
+	storeRequest(req);
+
+	return true;
+}
+
+void RsGxsDataAccess::generateToken(uint32_t &token)
+{
+	RsStackMutex stack(mDataMutex); /****** LOCKED *****/
+
+	token = mNextToken++;
+
+	return;
+}
+
+
+bool RsGxsDataAccess::requestMsgInfo(uint32_t &token, uint32_t ansType,
+		const RsTokReqOptions &opts, const GxsMsgReq &msgIds)
+{
+
+	GxsRequest* req = NULL;
+	uint32_t reqType = opts.mReqType;
+
+	if(reqType & GXS_REQUEST_TYPE_MSG_META)
+	{
+		MsgMetaReq* mmr = new MsgMetaReq();
+		mmr->mMsgIds = msgIds;
+		req = mmr;
+	}else if(reqType & GXS_REQUEST_TYPE_MSG_DATA)
+	{
+		MsgDataReq* mdr = new MsgDataReq();
+		mdr->mMsgIds = msgIds;
+		req = mdr;
+	}else if(reqType & GXS_REQUEST_TYPE_MSG_IDS)
+	{
+		MsgIdReq* mir = new MsgIdReq();
+		req = mir;
+	}
+
+	if(req == NULL)
+	{
+		std::cerr << "RsGxsDataAccess::requestMsgInfo() request type not recognised, type "
+				  << reqType << std::endl;
+		return false;
+	}else
+	{
+		generateToken(token);
+		std::cerr << "RsGxsDataAccess::requestMsgInfo() gets Token: " << token << std::endl;
+	}
+
+	setReq(req, token, ansType, opts);
+	storeRequest(req);
+	return true;
+}
+
+bool RsGxsDataAccess::requestSetGroupSubscribeFlags(uint32_t& token, const RsGxsGroupId &grpId, uint32_t subscribeFlags,
+		uint32_t subscribeMask)
+{
+
+	generateToken(token);
+
+	GroupSetFlagReq* req = new GroupSetFlagReq();
+
+	req->flag = subscribeFlags;
+	req->flagMask = subscribeMask;
+	req->grpId = grpId;
+
+	std::cerr << "RsGxsDataAccess::requestSetGroupSubscribeFlags() gets Token: " << token << std::endl;
+	storeRequest(req);
+
+	return false;
+}
+
+bool RsGxsDataAccess::requestSetGroupStatus(uint32_t& token, const RsGxsGroupId& grpId, uint32_t status, uint32_t statusMask)
+{
+
+	generateToken(token);
+
+	GroupSetFlagReq* req = new GroupSetFlagReq();
+
+	req->flag = status;
+	req->flagMask = statusMask;
+	req->grpId = grpId;
+
+	std::cerr << "RsGxsDataAccess::requestSetGroupStatus() gets Token: " << token << std::endl;
+	storeRequest(req);
+
+	return true;
+}
+
+bool RsGxsDataAccess::requestSetMessageStatus(uint32_t& token, const RsGxsGrpMsgIdPair &msgId, uint32_t status,
+		uint32_t statusMask)
+{
+
+	generateToken(token);
+
+	MessageSetFlagReq* req = new MessageSetFlagReq();
+
+	req->flag = status;
+	req->flagMask = statusMask;
+	req->msgId = msgId;
+
+	std::cerr << "RsGxsDataAccess::requestSetGroupStatus() gets Token: " << token << std::endl;
+	storeRequest(req);
+
+	return true;
+}
+
+void RsGxsDataAccess::setReq(GxsRequest* req, const uint32_t& token, const uint32_t& ansType, const RsTokReqOptions& opts) const
+{
+	req->token = token;
+	req->ansType = ansType;
+	req->Options = opts;
+	return;
+}
+void    RsGxsDataAccess::storeRequest(GxsRequest* req)
+{
+	RsStackMutex stack(mDataMutex); /****** LOCKED *****/
+
+	mRequests[req->token] = req;
+
+	return;
+}
+
+uint32_t RsGxsDataAccess::requestStatus(uint32_t token)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	return status;
+}
+
+
+
+
+
+bool RsGxsDataAccess::cancelRequest(const uint32_t& token)
+{
+	return clearRequest(token);
+}
+
+bool RsGxsDataAccess::clearRequest(const uint32_t& token)
+{
+	RsStackMutex stack(mDataMutex); /****** LOCKED *****/
+
+	std::map<uint32_t, GxsRequest*>::iterator it;
+
+	it = mRequests.find(token);
+	if (it == mRequests.end())
+	{
+		return false;
+	}
+
+	delete it->second;
+	mRequests.erase(it->first);
+
+	return true;
+}
+
+bool RsGxsDataAccess::getGroupSummary(const uint32_t& token, std::list<RsGxsGrpMetaData*>& groupInfo)
+{
+
+	GxsRequest* req = retrieveRequest(token);
+
+	if(req == NULL){
+
+		std::cerr << "RsGxsDataAccess::getGroupSummary() Unable to retrieve group summary" << std::endl;
+		return false;
+	}else  if(req->token == GXS_REQUEST_STATUS_COMPLETE){
+
+		GroupMetaReq* gmreq = dynamic_cast<GroupMetaReq*>(req);
+
+		if(gmreq)
+		{
+			groupInfo = gmreq->mGroupMetaData;
+			updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		}else{
+			std::cerr << "RsGxsDataAccess::getGroupSummary() Req found, failed caste" << std::endl;
+			return false;
+		}
+	}else{
+		std::cerr << "RsGxsDataAccess::getGroupSummary() Req not ready" << std::endl;
+		return false;
+	}
+
+	return true;
+}
+
+bool RsGxsDataAccess::getGroupData(const uint32_t& token, std::list<RsNxsGrp*>& grpData)
+{
+
+	GxsRequest* req = retrieveRequest(token);
+
+	if(req == NULL){
+
+		std::cerr << "RsGxsDataAccess::getGroupData() Unable to retrieve group data" << std::endl;
+		return false;
+	}else  if(req->token == GXS_REQUEST_STATUS_COMPLETE){
+
+		GroupDataReq* gmreq = dynamic_cast<GroupMetaReq*>(req);
+
+		if(gmreq)
+		{
+			grpData = gmreq->mGroupData;
+			updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		}else{
+			std::cerr << "RsGxsDataAccess::getGroupData() Req found, failed caste" << std::endl;
+			return false;
+		}
+	}else{
+		std::cerr << "RsGxsDataAccess::getGroupData() Req not ready" << std::endl;
+		return false;
+	}
+
+	return true;
+}
+
+bool RsGxsDataAccess::getMsgData(const uint32_t& token, NxsMsgDataResult& msgData)
+{
+	GxsRequest* req = retrieveRequest(token);
+
+	if(req == NULL){
+
+		std::cerr << "RsGxsDataAccess::getMsgData() Unable to retrieve group data" << std::endl;
+		return false;
+	}else if(req->token == GXS_REQUEST_STATUS_COMPLETE){
+
+		MsgDataReq* mdreq = dynamic_cast<GroupMetaReq*>(req);
+
+		if(mdreq)
+		{
+		 msgData = mdreq->mMsgData;
+		 updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		}else{
+			std::cerr << "RsGxsDataAccess::getMsgData() Req found, failed caste" << std::endl;
+			return false;
+		}
+	}else{
+		std::cerr << "RsGxsDataAccess::getMsgData() Req not ready" << std::endl;
+		return false;
+	}
+
+	return true;
+}
+
+bool RsGxsDataAccess::getMsgSummary(const uint32_t& token, GxsMsgMetaResult& msgInfo)
+{
+	GxsRequest* req = retrieveRequest(token);
+
+	if(req == NULL){
+
+		std::cerr << "RsGxsDataAccess::getMsgSummary() Unable to retrieve group data" << std::endl;
+		return false;
+	}else  if(req->token == GXS_REQUEST_STATUS_COMPLETE){
+
+		MsgMetaReq* mmreq = dynamic_cast<GroupMetaReq*>(req);
+
+		if(mmreq)
+		{
+		 msgInfo = mmreq->mMsgMetaData;
+		 updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+		}else{
+			std::cerr << "RsGxsDataAccess::getMsgSummary() Req found, failed caste" << std::endl;
+			return false;
+		}
+	}else{
+		std::cerr << "RsGxsDataAccess::getMsgSummary() Req not ready" << std::endl;
+		return false;
+	}
+
+	return true;
+}
+
+bool RsGxsDataAccess::getMsgList(const uint32_t& token, GxsMsgIdResult& msgIds)
+{
+	GxsRequest* req = retrieveRequest(token);
+
+	if(req == NULL){
+
+		std::cerr << "RsGxsDataAccess::getMsgList() Unable to retrieve group data" << std::endl;
+		return false;
+	}else  if(req->token == GXS_REQUEST_STATUS_COMPLETE){
+
+		MsgIdReq* mireq = dynamic_cast<GroupMetaReq*>(req);
+
+		if(mireq)
+		{
+		 msgIds = mireq->mMsgIdResult;
+		 updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+		}else{
+			std::cerr << "RsGxsDataAccess::getMsgList() Req found, failed caste" << std::endl;
+			return false;
+		}
+	}else{
+		std::cerr << "RsGxsDataAccess::getMsgList() Req not ready" << std::endl;
+		return false;
+	}
+
+	return true;
+}
+
+bool RsGxsDataAccess::getGroupList(const uint32_t& token, std::list<RsGxsGroupId>& groupIds)
+{
+	GxsRequest* req = retrieveRequest(token);
+
+	if(req == NULL){
+
+		std::cerr << "RsGxsDataAccess::getGroupList() Unable to retrieve group data,"
+				"\nRequest does not exist" << std::endl;
+		return false;
+	}else if(req->token == GXS_REQUEST_STATUS_COMPLETE){
+
+		GroupIdReq* gireq = dynamic_cast<GroupMetaReq*>(req);
+
+		if(gireq)
+		{
+		 groupIds = gireq->mGroupIdResult;
+		 updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+		}else{
+			std::cerr << "RsGxsDataAccess::getGroupList() Req found, failed caste" << std::endl;
+			return false;
+		}
+	}else{
+		std::cerr << "RsGxsDataAccess::getGroupList() Req not ready" << std::endl;
+		return false;
+	}
+
+	return true;
+}
+
+
+GxsRequest* RsGxsDataAccess::retrieveRequest(const uint32_t& token)
+{
+
+	RsStackMutex stack(mDataMutex);
+
+	if(mRequests.find(token) == mRequests.end()) return NULL;
+
+	GxsRequest* req = mRequests;
+
+	return req;
+}
+
+#define MAX_REQUEST_AGE 10
+
+void RsGxsDataAccess::processRequests()
+{
+
+	std::list<uint32_t> toClear;
+	std::list<uint32_t>::iterator cit;
+	time_t now = time(NULL);
+
+	{
+		RsStackMutex stack(mDataMutex); /******* LOCKED *******/
+
+		std::map<uint32_t, GxsRequest*>::iterator it;
+
+		GroupMetaReq* gmr;
+		GroupDataReq* gdr;
+		GroupIdReq* gir;
+
+		MsgMetaReq* mmr;
+		MsgDataReq* mdr;
+		MsgIdReq* mir;
+
+		for(it = mRequests.begin(); it != mRequests.end(); it++)
+		{
+
+			GxsRequest* req = it->second;
+			if (req->status == GXS_REQUEST_STATUS_PENDING)
+			{
+				std::cerr << "p3GxsDataService::fakeprocessrequests() Processing Token: " << req->token << " Status: "
+						<< req->status << " ReqType: " << req->reqType << " Age: "
+						<< now - req->reqTime << std::endl;
+
+				req->status = GXS_REQUEST_STATUS_PARTIAL;
+
+				/* PROCESS REQUEST! */
+
+				if((gmr = dynamic_cast<GroupMetaReq*>(req)) != NULL)
+				{
+					getGroupSummary(gmr);
+				}
+				else if((gdr = dynamic_cast<GroupDataReq*>(req)) != NULL)
+				{
+					getGroupData(gdr);
+				}
+				else if((gir = dynamic_cast<GroupIdReq*>(req)) != NULL)
+				{
+					getGroupList(gir);
+				}
+				else if((mmr = dynamic_cast<MsgMetaReq*>(req)) != NULL)
+				{
+					getMsgSummary(mmr);
+				}
+				else if((mdr = dynamic_cast<MsgDataReq*>(req)) != NULL)
+				{
+					getMsgData(mdr);
+				}
+				else if((mir = dynamic_cast<MsgIdReq*>(req)) != NULL)
+				{
+					getMsgList(mir);
+				}
+				else
+				{
+	#ifdef GXSDATA_SERVE_DEBUG
+					std::cerr << "RsGxsDataAccess::processRequests() Failed to process request, token: "
+							  << req->token << std::endl;
+	#endif
+
+					req->status = GXS_REQUEST_STATUS_FAILED;
+				}
+			}
+			else if (req->status == GXS_REQUEST_STATUS_PARTIAL)
+			{
+					req->status = GXS_REQUEST_STATUS_COMPLETE;
+			}
+			else if (req->status == GXS_REQUEST_STATUS_DONE)
+			{
+				std::cerr << "RsGxsDataAccess::processrequests() Clearing Done Request Token: "
+						  << req->token;
+				std::cerr << std::endl;
+				toClear.push_back(req->token);
+			}
+			else if (now - req->reqTime > MAX_REQUEST_AGE)
+			{
+				std::cerr << "RsGxsDataAccess::processrequests() Clearing Old Request Token: " << req->token;
+				std::cerr << std::endl;
+				toClear.push_back(req->token);
+			}
+		}
+
+	} // END OF MUTEX.
+
+	for(cit = toClear.begin(); cit != toClear.end(); cit++)
+	{
+		clearRequest(*cit);
+	}
+
+	return;
+}
+
+
+bool RsGxsDataAccess::getGroupData(GroupDataReq* req)
+{
+
+	std::map<RsGxsGroupId, RsNxsGrp*> grpData;
+	mDataStore->retrieveNxsGrps(grpData, true, true);
+
+	std::map<RsGxsGroupId, RsNxsGrp*>::iterator mit = grpData.begin();
+	for(; mit != grpData.end(); mit++)
+		req->mGroupData.push_back(mit->second);
+
+	return true;
+}
+
+bool RsGxsDataAccess::getGroupSummary(GroupMetaReq* req)
+{
+
+	std::map<RsGxsGroupId, RsGxsGrpMetaData*> grpMeta;
+
+	std::list<RsGxsGroupId>::const_iterator lit = req->mGroupIds.begin();
+
+	for(; lit != req->mGroupIds.end(); lit++)
+		grpMeta[*lit] = NULL;
+
+	mDataStore->retrieveGxsGrpMetaData(grpMeta);
+
+	std::map<RsGxsGroupId, RsGxsGrpMetaData*>::iterator mit = grpMeta.begin();
+
+	for(; mit != grpMeta.end(); mit++)
+		req->mGroupMetaData.push_back(mit->second);
+
+	return true;
+}
+
+bool RsGxsDataAccess::getGroupList(GroupIdReq* req)
+{
+	std::map<RsGxsGroupId, RsGxsGrpMetaData*> grpMeta;
+
+	std::list<RsGxsGroupId>::const_iterator lit = req->mGroupIds.begin();
+
+	for(; lit != req->mGroupIds.end(); lit++)
+		grpMeta[*lit] = NULL;
+
+	mDataStore->retrieveGxsGrpMetaData(grpMeta);
+
+	std::map<std::string, RsGxsGrpMetaData*>::iterator mit = grpMeta.begin();
+
+	for(; mit != grpMeta.end(); mit++)
+	{
+		req->mGroupIdResult.push_back(mit->first);
+		delete mit->second; // so wasteful!!
+	}
+
+	return true;
+}
+
+bool RsGxsDataAccess::getMsgData(MsgDataReq* req)
+{
+
+
+	GxsMsgResult result;
+	mDataStore->retrieveNxsMsgs(req->mMsgIds, result, true);
+
+	req->mMsgData = result;
+
+	return true;
+}
+
+
+bool RsGxsDataAccess::getMsgSummary(MsgMetaReq* req)
+{
+	GxsMsgMetaResult result;
+	std::vector<RsGxsGroupId> groupIds;
+	GxsMsgReq::iterator mit = req->mMsgIds.begin();
+	for(; mit != req->mMsgIds.end(); mit++)
+		groupIds.push_back(mit->first);
+
+	mDataStore->retrieveGxsMsgMetaData(groupIds, result);
+
+	req->mMsgMetaData = result;
+
+	return true;
+}
+
+bool RsGxsDataAccess::getMsgList(MsgIdReq* req)
+{
+	GxsMsgMetaResult result;
+	std::vector<RsGxsGroupId> groupIds;
+	GxsMsgReq::iterator mit = req->mMsgIds.begin();
+
+	const RsTokReqOptions& opts = req->Options;
+
+	for(; mit != req->mMsgIds.end(); mit++)
+		groupIds.push_back(mit->first);
+
+	{
+		RsStackMutex stack(mDataMutex);
+		mDataStore->retrieveGxsMsgMetaData(groupIds, result);
+	}
+
+
+	/* CASEs this handles.
+	 * Input is groupList + Flags.
+	 * 1) No Flags => All Messages in those Groups.
+	 *
+	 */
+	std::cerr << "RsGxsDataAccess::getMsgList()";
+	std::cerr << std::endl;
+
+
+	bool onlyOrigMsgs = false;
+	bool onlyLatestMsgs = false;
+	bool onlyThreadHeadMsgs = false;
+
+	// Can only choose one of these two.
+	if (opts.mOptions & RS_TOKREQOPT_MSG_ORIGMSG)
+	{
+		std::cerr << "RsGxsDataAccess::getMsgList() MSG_ORIGMSG";
+		std::cerr << std::endl;
+		onlyOrigMsgs = true;
+	}
+	else if (opts.mOptions & RS_TOKREQOPT_MSG_LATEST)
+	{
+		std::cerr << "RsGxsDataAccess::getMsgList() MSG_LATEST";
+		std::cerr << std::endl;
+		onlyLatestMsgs = true;
+	}
+
+	if (opts.mOptions & RS_TOKREQOPT_MSG_THREAD)
+	{
+		std::cerr << "RsGxsDataAccess::getMsgList() MSG_THREAD";
+		std::cerr << std::endl;
+		onlyThreadHeadMsgs = true;
+	}
+
+	GxsMsgMetaResult::iterator meta_it;
+	MsgMetaFilter metaFilter;
+
+	for(meta_it = result.begin(); meta_it != result.end(); meta_it++)
+	{
+		const RsGxsGroupId& grpId = meta_it->first;
+
+		metaFilter[grpId] = std::map<RsGxsMessageId, RsGxsMsgMetaData*>();
+
+		const std::vector<RsMsgMetaData*>& metaV = meta_it->second;
+		if (onlyLatestMsgs) // THIS ONE IS HARD -> LOTS OF COMP.
+		{
+			std::vector<RsMsgMetaData*>::const_iterator vit = metaV.begin();
+
+
+			// RUN THROUGH ALL MSGS... in map origId -> TS.
+			std::map<std::string, std::pair<std::string, time_t> > origMsgTs;
+			std::map<std::string, std::pair<std::string, time_t> >::iterator oit;
+
+			for(; vit != metaV.end(); vit++)
+			{
+				RsMsgMetaData* msgMeta = *vit;
+
+				/* if we are grabbing thread Head... then parentId == empty. */
+				if (onlyThreadHeadMsgs)
+				{
+					if (!(msgMeta->mParentId.empty()))
+					{
+						continue;
+					}
+				}
+
+
+				oit = origMsgTs.find(msgMeta->mOrigMsgId);
+				bool addMsg = false;
+				if (oit == origMsgTs.end())
+				{
+					std::cerr << "RsGxsDataAccess::getMsgList() Found New OrigMsgId: ";
+					std::cerr << msgMeta->mOrigMsgId;
+					std::cerr << " MsgId: " << msgMeta->mMsgId;
+					std::cerr << " TS: " << msgMeta->mPublishTs;
+					std::cerr << std::endl;
+
+					addMsg = true;
+				}
+				// check timestamps.
+				else if (oit->second.second < msgMeta->mPublishTs)
+				{
+					std::cerr << "RsGxsDataAccess::getMsgList() Found Later Msg. OrigMsgId: ";
+					std::cerr << msgMeta->mOrigMsgId;
+					std::cerr << " MsgId: " << msgMeta->mMsgId;
+					std::cerr << " TS: " << msgMeta->mPublishTs;
+
+					addMsg = true;
+				}
+
+				if (addMsg)
+				{
+					// add as latest. (overwriting if necessary)
+					origMsgTs[msgMeta->mOrigMsgId] = std::make_pair(msgMeta->mMsgId, msgMeta->mPublishTs);
+					metaFilter[grpId].insert(std::make_pair(msgMeta->mMsgId, msgMeta));
+				}
+			}
+
+			// Add the discovered Latest Msgs.
+			for(oit = origMsgTs.begin(); oit != origMsgTs.end(); oit++)
+			{
+				req->mMsgIds.insert(std::make_pair(grpId, oit->second.first));
+
+			}
+
+		}
+		else	// ALL OTHER CASES.
+		{
+			std::vector<RsMsgMetaData*>::const_iterator vit = metaV.begin();
+
+			for(; vit != metaV.end(); vit++)
+			{
+				RsMsgMetaData* msgMeta = *vit;
+				bool add = false;
+
+				/* if we are grabbing thread Head... then parentId == empty. */
+				if (onlyThreadHeadMsgs)
+				{
+					if (!(msgMeta->mParentId.empty()))
+					{
+						continue;
+					}
+				}
+
+
+				if (onlyOrigMsgs)
+				{
+					if (msgMeta->mMsgId == msgMeta->mOrigMsgId)
+					{
+						add = true;
+					}
+				}
+				else
+				{
+					add = true;
+				}
+
+				if (add)
+				{
+					req->mMsgIdResult.insert(grpId,msgMeta->mMsgId);
+				}
+
+			}
+		}
+	}
+
+	filterMsgList(req->mMsgIdResult, opts, metaFilter);
+	return true;
+}
+
+void RsGxsDataAccess::filterMsgList(GxsMsgIdResult& msgIds, const RsTokReqOptions& opts,
+		const MsgMetaFilter& msgMetas) const
+{
+
+	GxsMsgIdResult::iterator mit = msgIds.begin();
+	for(;mit != msgIds.end(); mit++)
+	{
+
+		MsgMetaFilter::const_iterator cit = msgMetas.find(mit->first);
+
+		if(cit == msgMetas.end())
+			continue;
+
+		std::vector<RsGxsMessageId>& msgs = mit->second;
+		std::vector<RsGxsMessageId>::iterator vit = msgs.begin();
+		const std::map<RsGxsMessageId, RsGxsMsgMetaData*>& meta = cit->second;
+		const std::map<RsGxsMessageId, RsGxsMsgMetaData*>::const_iterator cit2;
+
+		for(; vit != msgs.end();)
+		{
+
+			bool keep = false;
+			if( (cit2 = meta.find(*vit)) != meta.end() )
+			{
+				keep = checkMsgFilter(opts, cit2->second);
+			}
+
+			if(keep)
+			{
+				vit++;
+			}else
+			{
+				vit = msgs.erase(vit);
+			}
+		}
+	}
+}
+
+
+bool RsGxsDataAccess::checkRequestStatus(const uint32_t& token,
+		uint32_t& status, uint32_t& reqtype, uint32_t& anstype, time_t& ts)
+{
+
+	GxsRequest* req = retrieveRequest(token);
+
+	RsStackMutex stack(mDataMutex);
+
+	if(!req)
+		return false;
+
+	anstype = req->ansType;
+	reqtype = req->reqType;
+	status = req->status;
+	ts = req->reqTime;
+
+	return true;
+}
+
+bool RsGxsDataAccess::addGroupData(RsNxsGrp* grp) {
+
+	RsStackMutex stack(mDataMutex);
+
+	std::map<RsNxsGrp*, RsGxsGrpMetaData*> grpM;
+	grpM.insert(std::make_pair(grp, grp->metaData));
+	return mDataStore->storeGroup(grpM);
+}
+
+
+
+bool RsGxsDataAccess::addMsgData(RsNxsMsg* msg) {
+
+	RsStackMutex stack(mDataMutex);
+
+	std::map<RsNxsMsg*, RsGxsMsgMetaData*> msgM;
+	msgM.insert(std::make_pair(msg, msg->metaData));
+	return mDataStore->storeMessage(msgM);
+}
+
+
+
+void RsGxsDataAccess::tokenList(std::list<uint32_t>& tokens) {
+
+	RsStackMutex stack(mDataMutex);
+
+	std::map<uint32_t, GxsRequest*>::iterator mit = mRequests.begin();
+
+	for(; mit != mRequests.end(); mit++)
+	{
+		tokens.push_back(mit->first);
+	}
+}
+
+bool RsGxsDataAccess::checkMsgFilter(const RsTokReqOptions& opts, const RsGxsMsgMetaData* meta) const
+{
+	bool statusMatch = false;
+	if (opts.mStatusMask)
+	{
+		// Exact Flags match required.
+ 		if ((opts.mStatusMask & opts.mStatusFilter) == (opts.mStatusMask & meta->mMsgStatus))
+		{
+			std::cerr << "checkMsgFilter() Accepting Msg as StatusMatches: ";
+			std::cerr << " Mask: " << opts.mStatusMask << " StatusFilter: " << opts.mStatusFilter;
+			std::cerr << " MsgStatus: " << meta->mMsgStatus << " MsgId: " << meta->mMsgId;
+			std::cerr << std::endl;
+
+			statusMatch = true;
+		}
+		else
+		{
+			std::cerr << "checkMsgFilter() Dropping Msg due to !StatusMatch ";
+			std::cerr << " Mask: " << opts.mStatusMask << " StatusFilter: " << opts.mStatusFilter;
+			std::cerr << " MsgStatus: " << meta->mMsgStatus << " MsgId: " << meta->mMsgId;
+			std::cerr << std::endl;
+		}
+	}
+	else
+	{
+		// no status comparision,
+		statusMatch = true;
+	}
+	return statusMatch;
+}

libretroshare/src/gxs/rsgxsdataaccess.h

@@ -0,0 +1,323 @@
+#ifndef RSGXSDATAACCESS_H
+#define RSGXSDATAACCESS_H
+
+/*
+ * libretroshare/src/retroshare: rsgxsdataaccess.cc
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Robert Fernie, Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "rstokenservice.h"
+#include "rsgxsrequesttypes.h"
+#include "rsgds.h"
+
+
+typedef std::map< RsGxsGroupId, std::map<RsGxsMessageId, RsGxsMsgMetaData> > MsgMetaFilter;
+
+class RsGxsDataAccess : public RsTokenService
+{
+public:
+    RsGxsDataAccess(RsGeneralDataService* ds);
+    virtual ~RsGxsDataAccess() { return ;}
+
+public:
+
+    /** S: RsTokenService **/
+
+    /*!
+     *
+     * @param token
+     * @param ansType
+     * @param opts
+     * @param groupIds
+     * @return
+     */
+    bool requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<RsGxsGroupId> &groupIds);
+
+    /*!
+     * For requesting info on all messages of one or more groups
+     * @param token
+     * @param ansType
+     * @param opts
+     * @param groupIds
+     * @return
+     */
+    bool requestMsgInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const GxsMsgReq&);
+
+    /*!
+     * This sets the status of the message
+     * @param msgId the message id to set status for
+     * @param status status
+     * @param statusMask the mask for the settings targetted
+     * @return true if request made successfully, false otherwise
+     */
+    bool requestSetMessageStatus(uint32_t &token, const RsGxsGrpMsgIdPair &msgId,
+    		const uint32_t status, const uint32_t statusMask);
+
+    /*!
+     *
+     * @param token
+     * @param grpId
+     * @param status
+     * @param statusMask
+     * @return true if request made successfully, false otherwise
+     */
+    bool requestSetGroupStatus(uint32_t &token, const RsGxsGroupId &grpId, const uint32_t status,
+    		const uint32_t statusMask);
+
+    /*!
+     * Use request status to find out if successfully set
+     * @param groupId
+     * @param subscribeFlags
+     * @param subscribeMask
+     * @return true if request made successfully, false otherwise
+     */
+    bool requestSetGroupSubscribeFlags(uint32_t& token, const RsGxsGroupId &groupId, uint32_t subscribeFlags,
+    		uint32_t subscribeMask);
+
+
+    /* Poll */
+    uint32_t requestStatus(const uint32_t token);
+
+    /* Cancel Request */
+    bool cancelRequest(const uint32_t &token);
+
+    /** E: RsTokenService **/
+
+
+public:
+
+    /*!
+     * This adds a groups to the gxs data base, this is a blocking call
+     * Responsibility for grp still lies with callee \n
+     * If function returns successfully DataAccess can be queried for grp
+     * @param grp the group to add, responsibility grp passed lies with callee
+     * @return false if group cound not be added
+     */
+    bool addGroupData(RsNxsGrp* grp);
+
+    /*!
+     * This adds a group to the gxs data base, this is a blocking call \n
+     * Responsibility for msg still lies with callee \n
+     * If function returns successfully DataAccess can be queried for msg
+     * @param msg the msg to add
+     * @return false if msg could not be added, true otherwise
+     */
+    bool addMsgData(RsNxsMsg* msg);
+
+public:
+
+    /*!
+     * This must be called periodically to progress requests
+     */
+    void processRequests();
+
+    /*!
+     * Retrieve group list for a given token
+     * @param token request token to be redeemed
+     * @param groupIds
+     * @param msgIds
+     * @return false if token cannot be redeemed, if false you may have tried to redeem when not ready
+     */
+    bool getGroupList(const uint32_t &token, std::list<std::string> &groupIds);
+
+    /*!
+     *
+     * @param token request token to be redeemed
+     * @param msgIds
+     */
+    bool getMsgList(const uint32_t &token, GxsMsgIdResult &msgIds);
+
+
+    /*!
+     * @param token request token to be redeemed
+     * @param groupInfo
+     */
+    bool getGroupSummary(const uint32_t &token, std::list<RsGxsGrpMetaData*> &groupInfo);
+
+    /*!
+     *
+     * @param token request token to be redeemed
+     * @param msgInfo
+     */
+    bool getMsgSummary(const uint32_t &token, GxsMsgMetaResult &msgInfo);
+
+    /*!
+     *
+     * @param token request token to be redeemed
+     * @param grpData
+     */
+    bool getGroupData(const uint32_t &token, std::list<RsNxsGrp*>& grpData);
+
+    /*!
+     *
+     * @param token request token to be redeemed
+     * @param msgData
+     * @return false if data cannot be found for token
+     */
+    bool getMsgData(const uint32_t &token, NxsMsgDataResult& msgData);
+
+private:
+
+    /** helper functions to implement token service **/
+
+    /*!
+     * Assigns a token value to passed integer
+     * @param token is assigned a unique token value
+     */
+    void generateToken(uint32_t &token);
+
+    /*!
+     *
+     * @param token the value of the token for the request object handle wanted
+     * @return the request associated to this token
+     */
+    GxsRequest* retrieveRequest(const uint32_t& token);
+
+    /*!
+     * Add a gxs request to queue
+     * @param req gxs request to add
+     */
+    void storeRequest(GxsRequest* req);
+
+    /*!
+     * convenience function to setting members of request
+     * @param req
+     * @param token
+     * @param ansType
+     * @param opts
+     */
+    void setReq(GxsRequest* req,const uint32_t &token, const uint32_t& ansType, const RsTokReqOptions &opts) const;
+
+    /*!
+     * Remove request for request queue
+     * Request is deleted
+     * @param token the token associated to the request
+     * @return true if token successfully cleared, false if token does not exist
+     */
+    bool clearRequest(const uint32_t &token);
+
+    /*!
+     * Updates the status flag of a request
+     * @param token the token value of the request to set
+     * @param status the status to set
+     * @return
+     */
+    bool updateRequestStatus(const uint32_t &token, const uint32_t &status);
+
+    /*!
+     * Use to query the status and other values of a given token
+     * @param token the toke of the request to check for
+     * @param status set to current status of request
+     * @param reqtype set to request type of request
+     * @param anstype set to to anstype of request
+     * @param ts time stamp
+     * @return false if token does not exist, true otherwise
+     */
+    bool checkRequestStatus(const uint32_t &token, uint32_t &status, uint32_t &reqtype, uint32_t &anstype, time_t &ts);
+
+            // special ones for testing (not in final design)
+    /*!
+     * Get list of active tokens of this token service
+     * @param tokens sets to list of token contained in this tokenservice
+     */
+    void tokenList(std::list<uint32_t> &tokens);
+
+
+private:
+
+    /* These perform the actual blocking retrieval of data */
+
+    /*!
+     * Attempts to retrieve group id list from data store
+     * @param req
+     * @return false if unsuccessful, true otherwise
+     */
+    bool getGroupList(GroupIdReq* req);
+
+    /*!
+     * Attempts to retrieve msg id list from data store
+     * Computationally/CPU-Bandwidth expensive
+     * @param req
+     * @return false if unsuccessful, true otherwise
+     */
+    bool getMsgList(MsgIdReq* req);
+
+
+    /*!
+     * Attempts to retrieve group meta data from data store
+     * @param req
+     * @return false if unsuccessful, true otherwise
+     */
+    bool getGroupSummary(GroupMetaReq* req);
+
+    /*!
+     * Attempts to retrieve msg meta data from data store
+     * @param req
+     * @return false if unsuccessful, true otherwise
+     */
+    bool getMsgSummary(MsgMetaReq* req);
+
+    /*!
+     * Attempts to retrieve group data from data store
+     * @param req The request specifying data to retrieve
+     * @return false if unsuccessful, true otherwise
+     */
+    bool getGroupData(GroupDataReq* req);
+
+    /*!
+     * Attempts to retrieve message data from data store
+     * @param req The request specifying data to retrieve
+     * @return false if unsuccessful, true otherwise
+     */
+    bool getMsgData(MsgDataReq* req);
+
+    /*!
+     * This filter msgs based of options supplied (at the moment just status masks)
+     * @param msgIds The msgsIds to filter
+     * @param opts the request options set by user
+     * @param meta The accompanying meta information for msg, ids
+     */
+    void filterMsgList(GxsMsgIdResult& msgIds, const RsTokReqOptions& opts, const MsgMetaFilter& meta) const;
+
+
+    /*!
+     * This applies the options to the meta to find out if the given message satisfies
+     * them
+     * @param opts options containing filters to check
+     * @param meta meta containing currently defined options for msg
+     * @return true if msg meta passes all options
+     */
+    bool checkMsgFilter(const RsTokReqOptions& opts, const RsGxsMsgMetaData* meta) const;
+
+private:
+
+    RsGeneralDataService* mDataStore;
+	uint32_t mNextToken;
+	std::map<uint32_t, GxsRequest*> mRequests;
+
+    RsMutex mDataMutex;
+
+
+};
+
+#endif // RSGXSDATAACCESS_H

libretroshare/src/gxs/rsgxsflags.h

@@ -0,0 +1,55 @@
+#ifndef RSGXSFLAGS_H
+#define RSGXSFLAGS_H
+
+#include "inttypes.h"
+
+namespace GXS_SERV {
+
+
+    /*** GROUP FLAGS ***/
+
+    /* type of group */
+
+    static const uint32_t FLAG_GRP_TYPE_MASK;
+
+    // pub key encrypted
+    static const uint32_t FLAG_GRP_TYPE_PRIVATE;
+
+    // single publisher, read only
+    static const uint32_t FLAG_GRP_TYPE_RESTRICTED;
+
+    // anyone can publish
+    static const uint32_t FLAG_GRP_TYPE_PUBLIC;
+
+
+    /* type of msgs allowed */
+
+    static const uint32_t FLAG_MSG_TYPE_MASK;
+
+    // only signee can edit, and sign required
+    static const uint32_t FLAG_MSG_TYPE_SIGNED;
+
+    // no sign required, but signee can edit if signed
+    static const uint32_t FLAG_MSG_TYPE_ANON;
+
+    // anyone can mod but sign must be provided (needed for wikis)
+    static const uint32_t FLAG_MSG_TYPE_SIGNED_SHARED;
+
+    /*** GROUP FLAGS ***/
+
+
+
+    /*** MESSAGE FLAGS ***/
+
+    // indicates message edits an existing message
+    static const uint32_t FLAG_MSG_EDIT;
+
+    // indicates msg is id signed
+    static const uint32_t FLAG_MSG_ID_SIGNED;
+
+    /*** MESSAGE FLAGS ***/
+
+}
+
+
+#endif // RSGXSFLAGS_H

libretroshare/src/gxs/rsgxsnetservice.h

@@ -0,0 +1,377 @@
+#ifndef RSGXSNETSERVICE_H
+#define RSGXSNETSERVICE_H
+
+#include <list>
+#include <queue>
+
+#include "rsnxs.h"
+#include "rsgds.h"
+#include "rsnxsobserver.h"
+#include "pqi/p3linkmgr.h"
+#include "serialiser/rsnxsitems.h"
+
+#include "pqi/p3cfgmgr.h"
+
+
+/*!
+ * This represents a transaction made
+ * with the NxsNetService in all states
+ * of operation until completion
+ *
+ */
+class NxsTransaction
+{
+
+public:
+
+    static const uint8_t FLAG_STATE_STARTING; // when
+    static const uint8_t FLAG_STATE_RECEIVING; // begin receiving items for incoming trans
+    static const uint8_t FLAG_STATE_SENDING; // begin sending items for outgoing trans
+    static const uint8_t FLAG_STATE_COMPLETED;
+    static const uint8_t FLAG_STATE_FAILED;
+    static const uint8_t FLAG_STATE_WAITING_CONFIRM;
+
+    NxsTransaction();
+    ~NxsTransaction();
+
+    uint32_t mFlag; // current state of transaction
+    uint32_t mTimeOut;
+
+    /*!
+     * this contains who we
+     * c what peer this transaction involves.
+     * c The type of transaction
+     * c transaction id
+     * c timeout set for this transaction
+     * c and itemCount
+     */
+    RsNxsTransac* mTransaction;
+    std::list<RsNxsItem*> mItems; // items received or sent
+};
+
+/*!
+ * An abstraction of the net manager
+ * for retrieving Rs peers whom you will be synchronising
+ * and also you own Id
+ * Useful for testing also (abstracts away Rs's p3NetMgr)
+ */
+class RsNxsNetMgr
+{
+
+public:
+
+    virtual std::string getOwnId() = 0;
+    virtual void getOnlineList(std::set<std::string>& ssl_peers) = 0;
+
+};
+
+//class RsNxsNetMgrImpl : public RsNxsNetMgrImpl
+//{
+
+//public:
+
+//    RsNxsNetMgrImpl(p3LinkMgr* lMgr);
+
+//    std::string getOwnId();
+//    void getOnlineList(std::set<std::string>& ssl_peers);
+
+//};
+
+
+/// keep track of transaction number
+typedef std::map<uint32_t, NxsTransaction*> TransactionIdMap;
+
+/// to keep track of peers active transactions
+typedef std::map<std::string, TransactionIdMap > TransactionsPeerMap;
+
+
+/*!
+ * This class implements the RsNetWorkExchangeService
+ * using transactions to handle synchrnisation of Nxs items between
+ * peers in a network
+ * Transactions requires the maintenance of several states between peers, and whether
+ *
+ * Thus a data structure maintains: peers, and their active transactions
+ * Then for each transaction it needs to be noted if this is an outgoing or incoming transaction
+ * Outgoing transaction are in 3 different states:
+ *   1. START 2. INITIATED 3. SENDING 4. END
+ * Incoming transaction are also in 3 different states
+ *   1. START 2. RECEIVING 3. END
+ */
+class RsGxsNetService : public RsNetworkExchangeService, public p3ThreadedService,
+    public p3Config
+{
+public:
+
+
+    /*!
+     * only one observer is allowed
+     * @param servType service type
+     * @param gds The data service which allows read access to a service/store
+     * @param nxsObs observer will be notified whenever new messages/grps
+     * arrive
+     */
+    RsGxsNetService(uint16_t servType, RsGeneralDataService* gds, RsNxsNetMgr* netMgr, RsNxsObserver* nxsObs = NULL);
+
+    virtual ~RsGxsNetService();
+
+public:
+
+
+    /*!
+     * Use this to set how far back synchronisation of messages should take place
+     * @param age the max age a sync item can to be allowed in a synchronisation
+     */
+    void setSyncAge(uint32_t age);
+
+    /*!
+     * Explicitly requests all the groups contained by a peer
+     * Circumvents polling of peers for message
+     * @param peerId id of peer
+     */
+    void requestGroupsOfPeer(const std::string& peerId){ return;}
+
+    /*!
+     * get messages of a peer for a given group id, this circumvents the normal
+     * polling of peers for messages of given group id
+     * @param peerId Id of peer
+     * @param grpId id of group to request messages for
+     */
+    void requestMessagesOfPeer(const std::string& peerId, const std::string& grpId){ return; }
+
+    /*!
+     * pauses synchronisation of subscribed groups and request for group id
+     * from peers
+     * @param enabled set to false to disable pause, and true otherwise
+     */
+    void pauseSynchronisation(bool enabled);
+
+
+    /*!
+     * Request for this message is sent through to peers on your network
+     * and how many hops from them you've indicated
+     * @param msgId the messages to retrieve
+     * @return request token to be redeemed
+     */
+    int requestMsg(const std::string& msgId, uint8_t hops){ return 0;}
+
+    /*!
+     * Request for this group is sent through to peers on your network
+     * and how many hops from them you've indicated
+     * @param enabled set to false to disable pause, and true otherwise
+     * @return request token to be redeemed
+     */
+    int requestGrp(const std::list<std::string>& grpId, uint8_t hops){ return 0;}
+
+    /* p3Config methods */
+
+public:
+
+    bool	loadList(std::list<RsItem *>& load);
+    bool saveList(bool &cleanup, std::list<RsItem *>&);
+    RsSerialiser *setupSerialiser();
+
+public:
+
+    /*!
+     * initiates synchronisation
+     */
+    int tick();
+
+    /*!
+     * Processes transactions and job queue
+     */
+    void run();
+
+private:
+
+    /*!
+     * called when
+     * items are deemed to be waiting in p3Service item queue
+     */
+    void recvNxsItemQueue();
+
+
+    /** S: Transaction processing **/
+
+    /*!
+     * These process transactions which are in a wait state
+     * Also moves transaction which have been completed to
+     * the completed transactions list
+     */
+    void processTransactions();
+
+    /*!
+     * Process completed transaction, which either simply
+     * retires a transaction or additionally generates a response
+     * to the completed transaction
+     */
+    void processCompletedTransactions();
+
+    /*!
+     * Process transaction owned/started by user
+     * @param tr transaction to process, ownership stays with callee
+     */
+    void locked_processCompletedOutgoingTrans(NxsTransaction* tr);
+
+    /*!
+     * Process transactions started/owned by other peers
+     * @param tr transaction to process, ownership stays with callee
+     */
+    void locked_processCompletedIncomingTrans(NxsTransaction* tr);
+
+
+    /*!
+     * Process a transaction item, assumes a general lock
+     * @param item the transaction item to process
+     */
+    bool locked_processTransac(RsNxsTransac* item);
+
+    /*!
+     * This adds a transaction
+     * completeted transaction list
+     * If this is an outgoing transaction, transaction id is
+     * decrement
+     * @param trans transaction to add
+     */
+    void locked_completeTransaction(NxsTransaction* trans);
+
+    /*!
+     * This retrieves a unique transaction id that
+     * can be used in an outgoing transaction
+     */
+    uint32_t locked_getTransactionId();
+
+    /*!
+     * This attempts to push the transaction id counter back if you have
+     * active outgoing transactions in play
+     */
+    bool attemptRecoverIds();
+
+    /*!
+     * The cb listener is the owner of the grps
+     * @param grps
+     */
+    //void notifyListenerGrps(std::list<RsNxsGrp*>& grps);
+
+    /*!
+     * The cb listener is the owner of the msgs
+     * @param msgs
+     */
+    //void notifyListenerMsgs(std::list<RsNxsMsg*>& msgs);
+
+    /*!
+     * Generates new transaction to send msg requests based on list
+     * of msgs received from peer stored in passed transaction
+     * @param tr transaction responsible for generating msg request
+     */
+	void locked_genReqMsgTransaction(NxsTransaction* tr);
+
+    /*!
+     * Generates new transaction to send grp requests based on list
+     * of grps received from peer stored in passed transaction
+     * @param tr transaction responsible for generating grp request
+     */
+	void locked_genReqGrpTransaction(NxsTransaction* tr);
+
+	/*!
+	 * Generates new transaction to send msg data based on list
+	 * of grpids received from peer stored in passed transaction
+	 * @param tr transaction responsible for generating grp request
+	 */
+	void locked_genSendMsgsTransaction(NxsTransaction* tr);
+
+	/*!
+	 * Generates new transaction to send grp data based on list
+	 * of grps received from peer stored in passed transaction
+	 * @param tr transaction responsible for generating grp request
+	 */
+	void locked_genSendGrpsTransaction(NxsTransaction* tr);
+
+	/*!
+	 * convenience function to add a transaction to list
+	 * @param tr transaction to add
+	 */
+	bool locked_addTransaction(NxsTransaction* tr);
+
+	void cleanTransactionItems(NxsTransaction* tr) const;
+
+	/*!
+	 *  @param tr the transaction to check for timeout
+	 *  @return false if transaction has timed out, true otherwise
+	 */
+	bool locked_checkTransacTimedOut(NxsTransaction* tr);
+
+	/** E: Transaction processing **/
+
+    /** S: item handlers **/
+
+    /*!
+     * This attempts handles transaction items
+     * ownership of item is left with callee if this method returns false
+     * @param item transaction item to handle
+     * @return false if transaction could not be handled, ownership of item is left with callee
+     */
+    bool handleTransaction(RsNxsItem* item);
+
+    /*!
+     * Handles an nxs item for group synchronisation
+     * by startin a transaction and sending a list
+     * of groups held by user
+     * @param item contains grp sync info
+     */
+    void handleRecvSyncGroup(RsNxsSyncGrp* item);
+
+    /*!
+     * Handles an nxs item for msgs synchronisation
+     * @param item contaims msg sync info
+     */
+    void handleRecvSyncMessage(RsNxsSyncMsg* item);
+
+    /** E: item handlers **/
+
+
+    void syncWithPeers();
+
+private:
+
+    /*** transactions ***/
+
+    /// active transactions
+    TransactionsPeerMap mTransactions;
+
+    /// completed transactions
+    std::list<NxsTransaction*> mComplTransactions;
+
+    /// transaction id counter
+    uint32_t mTransactionN;
+
+    /*** transactions ***/
+
+    /*** synchronisation ***/
+    std::list<RsNxsSyncGrp*> mSyncGrp;
+    std::list<RsNxsSyncMsg*> mSyncMsg;
+    /*** synchronisation ***/
+
+    RsNxsObserver* mObserver;
+    RsGeneralDataService* mDataStore;
+    uint16_t mServType;
+
+    // how much time must elapse before a timeout failure
+    // for an active transaction
+    uint32_t mTransactionTimeOut;
+
+
+    std::string mOwnId;
+
+    RsNxsNetMgr* mNetMgr;
+
+    /// for other members save transactions
+    RsMutex mNxsMutex;
+
+    uint32_t mSyncTs;
+    const uint32_t mSYNC_PERIOD;
+
+};
+
+#endif // RSGXSNETSERVICE_H

libretroshare/src/gxs/rsgxsrequesttypes.h

@@ -0,0 +1,104 @@
+/*
+ * rsgxsrequesttypes.h
+ *
+ *  Created on: 21 Jul 2012
+ *      Author: crispy
+ */
+
+#ifndef RSGXSREQUESTTYPES_H_
+#define RSGXSREQUESTTYPES_H_
+
+
+class GxsRequest
+{
+
+public:
+	uint32_t token;
+	uint32_t reqTime;
+
+	uint32_t ansType;
+	uint32_t reqType;
+	RsTokReqOptions Options;
+
+	uint32_t status;
+};
+
+class GroupMetaReq : public GxsRequest
+{
+
+public:
+	std::list<std::string> mGroupIds;
+	std::list<RsGxsGrpMetaData*> mGroupMetaData;
+};
+
+class GroupIdReq : public GxsRequest
+{
+
+public:
+	std::list<std::string> mGroupIds;
+	std::list<std::string> mGroupIdResult;
+};
+
+class GroupDataReq : public GxsRequest
+{
+
+public:
+	std::list<std::string> mGroupIds;
+	std::list<RsNxsGrp*> mGroupData;
+};
+
+
+class MsgIdReq : public GxsRequest
+{
+
+public:
+
+	GxsMsgReq mMsgIds;
+	GxsMsgIdResult mMsgIdResult;
+};
+
+class MsgMetaReq : public GxsRequest
+{
+
+public:
+	GxsMsgReq mMsgIds;
+	GxsMsgMetaResult   mMsgMetaData;
+};
+
+class MsgDataReq : public GxsRequest
+{
+
+public:
+
+	GxsMsgReq mMsgIds;
+	NxsMsgDataResult mMsgData;
+};
+
+class GroupSetFlagReq : public GxsRequest
+{
+public:
+
+	const static uint32_t FLAG_SUBSCRIBE;
+	const static uint32_t FLAG_STATUS;
+
+	uint8_t type;
+	uint32_t flag;
+	uint32_t flagMask;
+	RsGxsGroupId grpId;
+
+};
+
+class MessageSetFlagReq : public GxsRequest
+{
+public:
+
+	const static uint32_t FLAG_STATUS;
+
+	uint8_t type;
+	uint32_t flag;
+	uint32_t flagMask;
+	RsGxsGrpMsgIdPair msgId;
+};
+
+
+#endif /* RSGXSREQUESTTYPES_H_ */

libretroshare/src/gxs/rsnxs.h

@@ -0,0 +1,132 @@
+#ifndef RSGNP_H
+#define RSGNP_H
+
+/*
+ * libretroshare/src/gxs: rsgnp.h
+ *
+ * Network Exchange Service interface for RetroShare.
+ *
+ * Copyright 2011-2011 by Robert Fernie, Christopher Evi-Prker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <set>
+#include <string>
+#include <time.h>
+#include <stdlib.h>
+#include <list>
+#include <map>
+
+#include "services/p3service.h"
+#include "rsgds.h"
+
+/*!
+ * Retroshare General Network Exchange Service: \n
+ * Interface:
+ *   - This provides a module to service peer's requests for GXS messages \n
+ *      and also request GXS messages from other peers. \n
+ *   - The general mode of operation is to sychronise all messages/grps between
+ *     peers
+ *
+ * The interface is sparse as this service is mostly making the requests to other GXS components
+ *
+ * Groups:
+ *   - As this is where exchanges occur between peers, this is also where group's relationships
+ *     should get resolved as far as
+ *   - Per implemented GXS there are a set of rules which will determine whether data is transferred
+ *     between any set of groups
+ *
+ *  1 allow transfers to any group
+ *  2 transfers only between group
+ *   - the also group matrix settings which is by default everyone can transfer to each other
+ */
+class RsNetworkExchangeService
+{
+public:
+
+	RsNetworkExchangeService(){ return;}
+
+    /*!
+     * Use this to set how far back synchronisation of messages should take place
+     * @param age the max age a sync item can to be allowed in a synchronisation
+     */
+    virtual void setSyncAge(uint32_t age) = 0;
+
+    /*!
+     * Explicitly requests all the groups contained by a peer
+     * Circumvents polling of peers for message
+     * @param peerId id of peer
+     */
+    virtual void requestGroupsOfPeer(const std::string& peerId) = 0;
+
+    /*!
+     * get messages of a peer for a given group id, this circumvents the normal
+     * polling of peers for messages of given group id
+     * @param peerId Id of peer
+     * @param grpId id of group to request messages for
+     */
+    virtual void requestMessagesOfPeer(const std::string& peerId, const std::string& grpId) = 0;
+
+    /*!
+     * Initiates a search through the network
+     * This returns messages which contains the search terms set in RsGxsSearch
+     * @param search contains search terms of requested from service
+     * @param hops how far into friend tree for search
+     * @return search token that can be redeemed later, implementation should indicate how this should be used
+     */
+    //virtual int searchMsgs(RsGxsSearch* search, uint8_t hops = 1, bool retrieve = 0) = 0;
+
+    /*!
+     * Initiates a search of groups through the network which goes
+     * a given number of hops deep into your friend network
+     * @param search contains search term requested from service
+     * @param hops number of hops deep into peer network
+     * @return search token that can be redeemed later
+     */
+    //virtual int searchGrps(RsGxsSearch* search, uint8_t hops = 1, bool retrieve = 0) = 0;
+
+
+    /*!
+     * pauses synchronisation of subscribed groups and request for group id
+     * from peers
+     * @param enabled set to false to disable pause, and true otherwise
+     */
+    virtual void pauseSynchronisation(bool enabled) = 0;
+
+
+    /*!
+     * Request for this message is sent through to peers on your network
+     * and how many hops from them you've indicated
+     * @param msgId the messages to retrieve
+     * @return request token to be redeemed
+     */
+    virtual int requestMsg(const std::string& msgId, uint8_t hops) = 0;
+
+    /*!
+     * Request for this group is sent through to peers on your network
+     * and how many hops from them you've indicated
+     * @param enabled set to false to disable pause, and true otherwise
+     * @return request token to be redeemed
+     */
+    virtual int requestGrp(const std::list<std::string>& grpId, uint8_t hops) = 0;
+
+
+};
+
+#endif // RSGNP_H

libretroshare/src/gxs/rsnxsobserver.h

@@ -0,0 +1,56 @@
+#ifndef RSNXSOBSERVER_H
+#define RSNXSOBSERVER_H
+
+/*
+ * libretroshare/src/gxp: gxp.h
+ *
+ * Observer, interface for RetroShare.
+ *
+ * Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <set>
+#include "serialiser/rsnxsitems.h"
+
+
+class RsNxsObserver
+{
+public:
+
+    RsNxsObserver() {}
+
+
+public:
+
+    /*!
+     * @param messages messages are deleted after function returns
+     */
+    virtual void notifyNewMessages(std::vector<RsNxsMsg*>& messages) = 0;
+
+    /*!
+     * @param messages messages are deleted after function returns
+     */
+    virtual void notifyNewGroups(std::vector<RsNxsGrp*>& groups) = 0;
+
+
+
+};
+
+#endif // RSNXSOBSERVER_H

libretroshare/src/gxs/rstokenservice.h

@@ -0,0 +1,184 @@
+#ifndef RSTOKENSERVICE_H
+#define RSTOKENSERVICE_H
+
+/*
+ * libretroshare/src/retroshare: rstokenservice.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Robert Fernie, Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+
+#include "serialiser/rsgxsitems.h"
+
+typedef std::map<RsGxsGroupId, std::vector<RsGxsMessageId> > GxsMsgReq;
+typedef std::map<RsGxsGroupId, std::vector<RsGxsMessageId> > GxsMsgIdResult;
+typedef std::map<RsGxsGroupId, std::vector<RsGxsMsgMetaData*> > GxsMsgMetaResult;
+typedef std::map<RsGxsGroupId, std::vector<RsNxsMsg*> > NxsMsgDataResult;
+
+#define GXS_REQUEST_TYPE_GROUP_DATA			0x00010000
+#define GXS_REQUEST_TYPE_GROUP_META			0x00020000
+#define GXS_REQUEST_TYPE_GROUP_IDS			0x00040000
+#define GXS_REQUEST_TYPE_MSG_DATA			0x00080000
+#define GXS_REQUEST_TYPE_MSG_META			0x00100000
+#define GXS_REQUEST_TYPE_MSG_IDS 			0x00200000
+
+/*!
+ * This class provides useful generic support for GXS style services.
+ * I expect much of this will be incorporated into the base GXS.
+ */
+class RsTokReqOptions
+{
+public:
+RsTokReqOptions()
+{
+	mOptions = 0;
+	mStatusFilter = 0; mStatusMask = 0; mSubscribeFilter = 0;
+	mBefore = 0; mAfter = 0; mReqType = 0;
+}
+
+uint32_t mOptions;
+
+// Request specific matches with Group / Message Status.
+// Should be usable with any Options... applied afterwards.
+uint32_t mStatusFilter;
+uint32_t mStatusMask;
+
+uint32_t mReqType;
+
+uint32_t mSubscribeFilter; // Only for Groups.
+
+// Time range... again applied after Options.
+time_t   mBefore;
+time_t   mAfter;
+};
+
+
+/*!
+ * A proxy class for requesting generic service data for GXS
+ * This seperates the request mechanism from the actual retrieval of data
+ */
+class RsTokenService
+{
+
+public:
+
+	static const uint8_t GXS_REQUEST_STATUS_FAILED;
+	static const uint8_t GXS_REQUEST_STATUS_PENDING;
+	static const uint8_t GXS_REQUEST_STATUS_PARTIAL;
+	static const uint8_t GXS_REQUEST_STATUS_FINISHED_INCOMPLETE;
+	static const uint8_t GXS_REQUEST_STATUS_COMPLETE;
+	static const uint8_t GXS_REQUEST_STATUS_DONE;			 // ONCE ALL DATA RETRIEVED.
+
+public:
+
+    RsTokenService()  { return; }
+    virtual ~RsTokenService() { return; }
+
+        /* Data Requests */
+
+    /*!
+     * Use this to request group related information
+     * @param token The token returned for the request, store this value to pool for request completion
+     * @param ansType The type of result (e.g. group data, meta, ids)
+     * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
+     * @param groupIds group id to request info for. Leave empty to get info on all groups,
+     * @return
+     */
+    virtual bool requestGroupInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<RsGxsGroupId> &groupIds) = 0;
+
+    /*!
+     * Use this to get msg related information, store this value to pole for request completion
+     * @param token The token returned for the request
+     * @param ansType The type of result wanted
+     * @param opts Additional option that affect outcome of request. Please see specific services, for valid values
+     * @param groupIds The ids of the groups to get, second entry of map empty to query for all msgs
+     * @return
+     */
+    virtual bool requestMsgInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const GxsMsgReq& msgIds) = 0;
+
+
+
+    /*!
+     * This sets the status of the message
+     * @param msgId the message id to set status for
+     * @param status status
+     * @param statusMask the mask for the settings targetted
+     * @return true if request made successfully, false otherwise
+     */
+    virtual bool requestSetMessageStatus(uint32_t &token, const RsGxsGrpMsgIdPair &msgId,
+    		const uint32_t status, const uint32_t statusMask) = 0;
+
+    /*!
+     * Set the status of a group given by group Id
+     * @param token The token returned for this request
+     * @param grpId The Id of the group to apply status change to
+     * @param status The status to apply
+     * @param statusMask The status mask (target particular type of status)
+     * @return true if request made successfully, false otherwise
+     */
+    virtual bool requestSetGroupStatus(uint32_t &token, const RsGxsGroupId &grpId, const uint32_t status,
+    		const uint32_t statusMask) = 0;
+
+    /*!
+     * Use request status to find out if successfully set
+     * @param groupId
+     * @param subscribeFlags
+     * @param subscribeMask
+     * @return true if request made successfully, false otherwise
+     */
+    virtual bool requestSetGroupSubscribeFlags(uint32_t& token, const RsGxsGroupId &groupId, uint32_t subscribeFlags, uint32_t subscribeMask) = 0;
+
+
+    	// (FUTURE WORK).
+    //virtual bool groupRestoreKeys(const std::string &groupId) = 0;
+    //virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers) = 0;
+
+        /* Poll */
+
+    /*!
+     * Request the status of ongoing request.
+     * Please use this for polling as much cheaper
+     * than polling the specific service as they might
+     * not return intermediate status information
+     * @param token value of token to check status for
+     * @return the current status of request
+     */
+    virtual uint32_t requestStatus(const uint32_t token) = 0;
+
+        /* Cancel Request */
+
+    /*!
+     * If this function returns false, it may be that the request has completed
+     * already. Useful for very expensive request. This is a blocking operation
+     * @param token the token of the request to cancel
+     * @return false if unusuccessful in cancelling request, true if successful
+     */
+    virtual bool cancelRequest(const uint32_t &token) = 0;
+
+
+
+};
+
+#endif // RSTOKENSERVICE_H

libretroshare/src/libretroshare.pro

@@ -7,6 +7,10 @@ TARGET = retroshare
 
 CONFIG += test_voip 
 
+# GXS Stuff.
+#CONFIG += newcache
+#CONFIG += newservices
+
 # Beware: All data of the stripped services are lost
 DEFINES *= PQI_DISABLE_TUNNEL 
 #ENABLE_CACHE_OPT
@@ -416,7 +420,6 @@ HEADERS +=	rsserver/p3discovery.h \
 			rsserver/p3history.h \
 			rsserver/p3msgs.h \
 			rsserver/p3peers.h \
-			rsserver/p3photo.h \
 			rsserver/p3status.h \
 			rsserver/p3serverconfig.h
 
@@ -431,7 +434,6 @@ HEADERS +=	serialiser/rsbaseitems.h \
 			serialiser/rsgameitems.h \
 			serialiser/rshistoryitems.h \
 			serialiser/rsmsgitems.h \
-			serialiser/rsphotoitems.h \
 			serialiser/rsserial.h \
 			serialiser/rsserviceids.h \
 			serialiser/rsserviceitems.h \
@@ -456,7 +458,6 @@ HEADERS +=	services/p3channels.h \
 			services/p3gamelauncher.h \
 			services/p3gameservice.h \
 			services/p3msgservice.h \
-			services/p3photoservice.h \
 			services/p3service.h \
 			services/p3statusservice.h \
 			services/p3dsdv.h \
@@ -488,9 +489,8 @@ HEADERS +=	util/folderiterator.h \
 			util/rsversion.h \
 			util/rswin.h \
 			util/rsrandom.h \
-			util/pugiconfig.h \
 			util/radix64.h \
-			util/pugixml.h
+			util/pugiconfig.h \  
 
 SOURCES +=	dbase/cachestrapper.cc \
 			dbase/fimonitor.cc \
@@ -553,7 +553,6 @@ SOURCES +=	rsserver/p3discovery.cc \
 			rsserver/p3history.cc \
 			rsserver/p3msgs.cc \
 			rsserver/p3peers.cc \
-			rsserver/p3photo.cc \
 			rsserver/p3status.cc \
 			rsserver/rsiface.cc \
 			rsserver/rsinit.cc \
@@ -576,7 +575,6 @@ SOURCES +=	serialiser/rsbaseitems.cc \
 			serialiser/rsgameitems.cc \
 			serialiser/rshistoryitems.cc \
 			serialiser/rsmsgitems.cc \
-			serialiser/rsphotoitems.cc \
 			serialiser/rsserial.cc \
 			serialiser/rsstatusitems.cc \
 			serialiser/rstlvaddrs.cc \
@@ -600,7 +598,6 @@ SOURCES +=	services/p3channels.cc \
 			services/p3forums.cc \
 			services/p3gamelauncher.cc \
 			services/p3msgservice.cc \
-			services/p3photoservice.cc \
 			services/p3service.cc \
 			services/p3statusservice.cc \
 			services/p3dsdv.cc \
@@ -634,7 +631,6 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsversion.cc \
 			util/rswin.cc \
 			util/rsrandom.cc \
-			util/pugixml.cc
 
 zeroconf {
 
@@ -649,7 +645,7 @@ SOURCES +=	zeroconf/p3zeroconf.cc  \
 
 # This is seperated from the above for windows/linux platforms.
 # It is acceptable to build in zeroconf and have it not work, 
-# but unacceptable to rely on Apple's libraries for Upnp when we have alternatives.
+# but unacceptable to rely on Apple's libraries for Upnp when we have alternatives. '
 
 zcnatassist {
 
@@ -661,3 +657,67 @@ SOURCES +=	zeroconf/p3zcnatassist.cc \
 
 }
 
+ # new gxs cache system
+newcache { 
+
+HEADERS += serialiser/rsnxsitems.h \
+            gxs/rsgds.h \
+            gxs/rsgxs.h \
+            gxs/rsdataservice.h \
+            gxs/rsgxsnetservice.h \
+            gxs/rsgxsflags.h \
+            gxs/rsgenexchange.h \
+            gxs/rsnxsobserver.h \
+            gxs/rsgxsdata.h \
+            gxs/rstokenservice.h \
+            gxs/rsgxsdataaccess.h \
+    		retroshare/rsgxsservice.h \
+    		serialiser/rsgxsitems.h \
+		util/retrodb.h
+
+SOURCES += serialiser/rsnxsitems.cc \
+                gxs/rsdataservice.cc \
+                gxs/rsgenexchange.cc \
+            gxs/rsgxsnetservice.cc \
+            gxs/rsgxsdata.cc \
+    		services/p3photoserviceV2.cc \
+        	gxs/rsgxsdataaccess.cc \
+		util/retrodb.cc
+}
+
+
+
+newservices { 
+
+HEADERS += services/p3photoservice.h \
+		serialiser/rsphotoitems.h \
+		retroshare/rsphoto.h \
+		services/p3gxsservice.h \
+		retroshare/rsidentity.h \
+		services/p3wikiservice.h \
+		retroshare/rswiki.h \
+		retroshare/rswire.h \
+		services/p3wire.h \
+		services/p3idservice.h \
+		retroshare/rsforumsv2.h \
+		services/p3forumsv2.h \
+		retroshare/rsposted.h \
+		services/p3posted.h \
+    	services/p3photoserviceV2.h \
+    	retroshare/rsphotoV2.h \
+    	
+
+SOURCES += services/p3photoservice.cc \
+		serialiser/rsphotoitems.cc \
+		services/p3gxsservice.cc \
+		services/p3wikiservice.cc \
+		services/p3wire.cc \
+		services/p3idservice.cc \
+		services/p3forumsv2.cc \
+		services/p3posted.cc \
+
+# Other Old Code.
+#	rsserver/p3photo.cc \
+}
+
+

libretroshare/src/retroshare/rsforumsv2.h

@@ -0,0 +1,106 @@
+#ifndef RETROSHARE_FORUMV2_GUI_INTERFACE_H
+#define RETROSHARE_FORUMV2_GUI_INTERFACE_H
+
+/*
+ * libretroshare/src/retroshare: rsforumv2.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+
+#include <retroshare/rsidentity.h>
+
+/* The Main Interface Class - for information about your Peers */
+class RsForumsV2;
+extern RsForumsV2 *rsForumsV2;
+
+class RsForumV2Group
+{
+	public:
+
+	// All the MetaData is Stored here:
+	RsGroupMetaData mMeta;
+
+	// THESE ARE IN THE META DATA.
+	//std::string mGroupId;
+	//std::string mName;
+
+	std::string mDescription;
+
+	// THESE ARE CURRENTLY UNUSED.
+	//std::string mCategory;
+	//std::string mHashTags;
+};
+
+class RsForumV2Msg
+{
+	public:
+
+	// All the MetaData is Stored here:
+	RsMsgMetaData mMeta;
+
+	// THESE ARE IN THE META DATA.
+	//std::string mGroupId;
+	//std::string mMsgId;
+	//std::string mOrigMsgId;
+	//std::string mThreadId;
+	//std::string mParentId;
+	//std::string mName;      (aka. Title)
+
+	std::string mMsg; // all the text is stored here.
+
+	// THESE ARE CURRENTLY UNUSED.
+	//std::string mHashTags;
+};
+
+class RsForumsV2: public RsTokenService
+{
+	public:
+
+	RsForumsV2()  { return; }
+virtual ~RsForumsV2() { return; }
+
+	/* Specific Service Data */
+virtual bool getGroupData(const uint32_t &token, RsForumV2Group &group) = 0;
+virtual bool getMsgData(const uint32_t &token, RsForumV2Msg &msg) = 0;
+
+
+	// ONES THAT WE ARE NOT IMPLEMENTING. (YET!)
+//virtual bool getMessageStatus(const std::string& fId, const std::string& mId, uint32_t& status) = 0;
+
+// THINK WE CAN GENERALISE THIS TO: a list function, and you can just count the list entries...
+// requestGroupList(groupId, UNREAD, ...)
+//virtual bool getMessageCount(const std::string &groupId, unsigned int &newCount, unsigned int &unreadCount) = 0;
+
+
+/* details are updated in group - to choose GroupID */
+virtual bool createGroup(uint32_t &token, RsForumV2Group &group, bool isNew) = 0;
+virtual bool createMsg(uint32_t &token, RsForumV2Msg &msg, bool isNew) = 0;
+
+};
+
+
+
+#endif

libretroshare/src/retroshare/rsgxsservice.h

@@ -0,0 +1,45 @@
+#ifndef RSGXSSERVICE_H
+#define RSGXSSERVICE_H
+
+
+#include "gxs/rstokenservice.h"
+
+
+/*!
+ * The aim of this class is to abstract
+ * how changes are represented so
+ * they can be determined outside the
+ * client API without explcitly
+ * enumerating all possible changes
+ * at the interface
+ */
+class RsGxsChange
+{
+public:
+    RsGxsChange(){ return; }
+
+};
+
+/*!
+ * Relevant to group changes
+ * TODO: extent to indicate whether a meta change or actual data
+ */
+class RsGxsGroupChange : RsGxsChange
+{
+public:
+    std::list<RsGxsGroupId> grpIdList;
+};
+
+/*!
+ * Relevant to message changes
+ * TODO: extent to indicate whether a meta change or actual data
+ */
+class RsGxsMsgChange : RsGxsChange
+{
+public:
+    std::map<RsGxsGroupId, std::vector<RsGxsMessageId> > msgChangeMap;
+};
+
+
+
+#endif // RSGXSSERVICE_H

libretroshare/src/retroshare/rsidentity.h

@@ -0,0 +1,497 @@
+#ifndef RETROSHARE_IDENTITY_GUI_INTERFACE_H
+#define RETROSHARE_IDENTITY_GUI_INTERFACE_H
+
+/*
+ * libretroshare/src/retroshare: rsidentity.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+
+// FLAGS WILL BE REUSED FROM RSDISTRIB -> FOR NOW.
+#include <retroshare/rsdistrib.h>
+
+/********** Generic Token Request Interface ***********************
+ * This is packaged here, as most TokenServices will require ID Services too.
+ * The requests can be generic, but the reponses are service specific (dependent on data types).
+ */
+
+// This bit will be filled out over time.
+#define RS_TOKREQOPT_MSG_VERSIONS	0x0001		// MSGRELATED: Returns All MsgIds with OrigMsgId = MsgId.
+#define RS_TOKREQOPT_MSG_ORIGMSG	0x0002		// MSGLIST: All Unique OrigMsgIds in a Group.
+#define RS_TOKREQOPT_MSG_LATEST		0x0004		// MSGLIST: All Latest MsgIds in Group. MSGRELATED: Latest MsgIds for Input Msgs.
+
+#define RS_TOKREQOPT_MSG_THREAD		0x0010		// MSGRELATED: All Msgs in Thread. MSGLIST: All Unique Thread Ids in Group. 
+#define RS_TOKREQOPT_MSG_PARENT		0x0020		// MSGRELATED: All Children Msgs.
+
+#define RS_TOKREQOPT_MSG_AUTHOR		0x0040		// MSGLIST: Messages from this AuthorId 
+
+
+// Status Filtering... should it be a different Option Field.
+#define RS_TOKREQOPT_GROUP_UPDATED	0x0100		// GROUPLIST: Groups that have been updated.
+#define RS_TOKREQOPT_MSG_UPDATED	0x0200		// MSGLIST: Msg that have been updated from specified groups.
+#define RS_TOKREQOPT_MSG_UPDATED	0x0200		// MSGLIST: Msg that have been updated from specified groups.
+
+
+
+// Read Status.
+#define RS_TOKREQOPT_READ		0x0001
+#define RS_TOKREQOPT_UNREAD		0x0002
+
+#define RS_TOKREQ_ANSTYPE_LIST		0x0001		
+#define RS_TOKREQ_ANSTYPE_SUMMARY	0x0002		
+#define RS_TOKREQ_ANSTYPE_DATA		0x0003		
+
+
+
+
+class RsTokReqOptions
+{
+	public:
+	RsTokReqOptions() 
+	{ 
+		mOptions = 0; 
+		mStatusFilter = 0; mStatusMask = 0; mSubscribeFilter = 0; 
+		mBefore = 0; mAfter = 0; 
+	}
+
+	uint32_t mOptions;
+
+	// Request specific matches with Group / Message Status.
+	// Should be usable with any Options... applied afterwards.
+	uint32_t mStatusFilter;
+	uint32_t mStatusMask;
+
+	uint32_t mSubscribeFilter; // Only for Groups.
+
+	// Time range... again applied after Options.
+	time_t   mBefore;
+	time_t   mAfter;
+};
+
+
+/*********************************************************
+ * Documentation for Groups Definitions.
+ * 
+ * A Group is defined by:
+ *	- TWO RSA Keys. (Admin Key & Publish Key)
+ *	- Publish TS: Used to select the latest definition.
+ *
+ *	- Operating Mode:
+ *		- Circle (Public, External, Private).
+ *		- Publish Mode: Encrypted / All-Signed / Only ThreadHead / None Required.
+ *		- AuthorId: GPG Required / Any Required / Only if no Publish Signature.
+ *
+ *	- Description:
+ *		- Name & Description.
+ *		- Optional AuthorId.
+ * 
+ * Most of this information is contained inside the GroupMetaData.
+ * except for Actual Admin / Publish Keys, which are maintained internally.
+ *
+ *******
+ *      - Group Definition must be signed by Admin Key, otherwise invalid.
+ *	- Circle Definition controls distribution of Group and Messages, see section on this for more details. 
+ *	- Public parts of Keys are distributed with Definition.
+ *	- Private parts can be distributed to select people via alternative channels.
+ *      - A Message Requires at least one signature: publish or Author. This signature will be used as MsgId.
+ *
+ *	Groups will operate in the following modes:
+ *	1) Public Forum: PublishMode = None Required, AuthorId: Required.  
+ *	2) Closed Forum: PublishMode = All-Signed, AuthorId: Required.  
+ *	3) Private Forum: PublishMode = Encrypted, AuthorId: Required.  
+ *
+ *      4) Anon Channel: PublishMode = All-Signed, AuthorId: None.
+ *      5) Anon Channel with Comments: PublishMode = Only ThreadHead, AuthorId: If No Publish Signature.
+ *      6) Private Channel: PublishMode = Encrypted. 
+ *
+ *      7) Personal Photos - with comments: PublishMode = Only ThreadHead, AuthorId: Required.
+ *      8) Personal Photos - no comments: PublishMode = All-Signed, AuthorId: Required.
+ *
+ *      9 ) Public  Wiki: PublishMode = None Required, AuthorId: Required.
+ *      10) Closed  Wiki: PublishMode = All-Signed, AuthorId: Required.
+ *      11) Private Wiki: PublishMode = Encrypted, AuthorId: Required.
+ *
+ *      12) Twitter: PublishMode = Only ThreadHead, AuthorId: Required.
+ *
+ *      13) Posted: PublishMode = None Required, AuthorId: Required.
+ *
+ *
+ ******
+ *
+ * Additionally to this information. The MetaData also contains several fields which can 
+ * be used to store local information for the benefit of the service.
+ *
+ * In Particular: MsgStatus & GroupStatus inform the service if the user has read the message or if anything has changed.
+ *
+ ***/
+
+
+// Control of Publish Signatures.
+#define RSGXS_GROUP_SIGN_PUBLISH_MASK  		0x000000ff
+#define RSGXS_GROUP_SIGN_PUBLISH_ENCRYPTED 	0x00000001
+#define RSGXS_GROUP_SIGN_PUBLISH_ALLSIGNED 	0x00000002
+#define RSGXS_GROUP_SIGN_PUBLISH_THREADHEAD	0x00000004
+#define RSGXS_GROUP_SIGN_PUBLISH_NONEREQ	0x00000008
+
+// Author Signature.
+#define RSGXS_GROUP_SIGN_AUTHOR_MASK  		0x0000ff00
+#define RSGXS_GROUP_SIGN_AUTHOR_GPG 		0x00000100
+#define RSGXS_GROUP_SIGN_AUTHOR_REQUIRED 	0x00000200
+#define RSGXS_GROUP_SIGN_AUTHOR_IFNOPUBSIGN	0x00000400
+#define RSGXS_GROUP_SIGN_AUTHOR_NONE		0x00000800
+
+// NB: That one signature is required...
+// so some combinations are not possible. e.g.
+// SIGN_PUBLISH_NONEREQ	&& SIGN_AUTHOR_NONE is not allowed.
+// SIGN_PUBLISH_THREADHEAD && SIGN_AUTHOR_NONE is also invalid.
+
+#define RSGXS_GROUP_SIGN_RESERVED_MASK  	0xffff0000
+
+
+// STATUS FLAGS: There is space here for Service specific flags - if they so desire.
+//
+// Msgs: UNREAD_BY_USER & PROCESSED are useful.
+// Groups: NEW_MESSAGES & GROUP_UPDATED.
+
+#define RSGXS_MSG_STATUS_MASK           0x0000000f
+#define RSGXS_MSG_STATUS_READ 		0x00000001	// New or Not New
+#define RSGXS_MSG_STATUS_UNREAD_BY_USER 0x00000002
+#define RSGXS_MSG_STATUS_UNPROCESSED	0x00000004	// By the Service.
+
+#define RSGXS_MSG_STATUS_SERVICE_MASK 	0xffff0000
+
+#define RSGXS_GROUP_STATUS_MASK  	0x0000000f
+#define RSGXS_GROUP_STATUS_UPDATED 	0x00000001
+#define RSGXS_GROUP_STATUS_NEWGROUP	0x00000002	
+
+#define RSGXS_GROUP_STATUS_SERVICE_MASK 0xffff0000
+
+
+
+// Subscription Flags. (LOCAL)
+#define RSGXS_GROUP_SUBSCRIBE_MASK  		0x0000000f
+#define RSGXS_GROUP_SUBSCRIBE_ADMIN  		0x00000001
+#define RSGXS_GROUP_SUBSCRIBE_PUBLISH  		0x00000002
+#define RSGXS_GROUP_SUBSCRIBE_SUBSCRIBED	0x00000004
+#define RSGXS_GROUP_SUBSCRIBE_MONITOR		0x00000008
+
+
+// Some MACROS for EASE OF USE. (USED BY FORUMSV2 At the moment.
+#define IS_MSG_UNREAD(status) 			((status & RSGXS_MSG_STATUS_READ) == 0 || (status & RSGXS_MSG_STATUS_UNREAD_BY_USER))
+#define IS_GROUP_ADMIN(subscribeFlags) 		(subscribeFlags & RSGXS_GROUP_SUBSCRIBE_ADMIN)
+#define IS_GROUP_SUBSCRIBED(subscribeFlags) 	(subscribeFlags & (RSGXS_GROUP_SUBSCRIBE_ADMIN | RSGXS_GROUP_SUBSCRIBE_SUBSCRIBED))
+
+
+
+#define RSGXS_MAX_SERVICE_STRING	200 // Sensible limit for dbase usage.
+
+
+class RsGroupMetaData
+{
+	public:
+
+	RsGroupMetaData()
+	{
+		mGroupFlags = 0;
+		mSignFlags = 0;
+		mSubscribeFlags = 0;
+
+		mPop = 0;
+		mMsgCount = 0;
+		mLastPost = 0;
+		mGroupStatus = 0;
+	
+		//mPublishTs = 0;
+	}
+	
+	std::string mGroupId;
+	std::string mGroupName;
+	uint32_t    mGroupFlags;  // Service Specific Options ????
+	uint32_t    mSignFlags;   // Combination of RSGXS_GROUP_SIGN_PUBLISH_MASK & RSGXS_GROUP_SIGN_AUTHOR_MASK.
+
+	time_t      mPublishTs; // Mandatory.
+	std::string mAuthorId;   // Optional. 
+
+	// BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.
+
+	uint32_t    mSubscribeFlags;
+
+	uint32_t    mPop; // HOW DO WE DO THIS NOW.
+	uint32_t    mMsgCount; // ???
+	time_t      mLastPost; // ???
+
+	uint32_t    mGroupStatus;
+
+	std::string mServiceString; // Service Specific Free-Form extra storage.
+};
+
+
+
+
+class RsMsgMetaData
+{
+	public:
+
+	RsMsgMetaData()
+	{
+		mPublishTs = 0;
+		mMsgFlags = 0;
+		mMsgStatus = 0;
+		mChildTs = 0;
+	}
+
+	std::string mGroupId;
+	std::string mMsgId;
+
+	std::string mThreadId;
+	std::string mParentId;
+	std::string mOrigMsgId;
+
+	std::string mAuthorId;
+	
+	std::string mMsgName;
+	time_t      mPublishTs;
+
+	uint32_t    mMsgFlags; // Whats this for? (Optional Service Specific - e.g. flag MsgType)
+
+	// BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.
+	// normally READ / UNREAD flags. LOCAL Data.
+	uint32_t    mMsgStatus;
+	time_t      mChildTs;
+
+	std::string mServiceString; // Service Specific Free-Form extra storage.
+
+};
+
+std::ostream &operator<<(std::ostream &out, const RsGroupMetaData &meta);
+std::ostream &operator<<(std::ostream &out, const RsMsgMetaData &meta);
+
+class RsTokenService
+{
+	public:
+
+	RsTokenService()  { return; }
+virtual ~RsTokenService() { return; }
+
+        /* changed? */
+virtual bool updated() = 0;
+
+        /* Data Requests */
+virtual bool requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds) = 0;
+virtual bool requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds) = 0;
+virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds) = 0;
+
+        /* Generic Lists */
+virtual bool getGroupList(         const uint32_t &token, std::list<std::string> &groupIds) = 0;
+virtual bool getMsgList(           const uint32_t &token, std::list<std::string> &msgIds) = 0;
+
+        /* Generic Summary */
+virtual bool getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo) = 0;
+virtual bool getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo) = 0;
+
+	/* Actual Data -> specific to Interface */
+
+
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token) = 0;
+
+	/* Cancel Request */
+virtual bool cancelRequest(const uint32_t &token) = 0;
+
+
+	//////////////////////////////////////////////////////////////////////////////
+	/* Functions from Forums -> need to be implemented generically */
+	// Groups Changed is now part of requestGroupInfo request.
+//virtual bool groupsChanged(std::list<std::string> &groupIds) = 0;
+
+        // Message/Group Status - is retrived via requests...
+        // These operations could have a token, but for the moment we are going to assume
+        // they are async and always succeed - (or fail silently).
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask) = 0;
+virtual bool setGroupStatus(const std::string &grpId, const uint32_t status, const uint32_t statusMask) = 0;
+
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask) = 0;
+
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str) = 0;
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str) = 0;
+
+	// (FUTURE WORK).	
+virtual bool groupRestoreKeys(const std::string &groupId) = 0;
+virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers) = 0;
+
+
+
+
+};
+
+
+
+
+/* The Main Interface Class - for information about your Peers */
+class RsIdentity;
+extern RsIdentity *rsIdentity;
+
+#define RSID_TYPE_MASK		0xff00
+#define RSID_RELATION_MASK	0x00ff
+
+#define RSID_TYPE_REALID	0x0100
+#define RSID_TYPE_PSEUDONYM	0x0200
+
+#define RSID_RELATION_YOURSELF  0x0001
+#define RSID_RELATION_FRIEND	0x0002
+#define RSID_RELATION_FOF	0x0004
+#define RSID_RELATION_OTHER   	0x0008
+#define RSID_RELATION_UNKNOWN 	0x0010
+
+std::string rsIdTypeToString(uint32_t idtype);
+
+class RsIdGroup
+{
+	public:
+
+
+	RsGroupMetaData mMeta;
+
+	// In GroupMetaData.
+	//std::string mNickname; (mGroupName)
+	//std::string mKeyId;    (mGroupId)
+
+	uint32_t mIdType;
+
+	std::string mGpgIdHash; // SHA(KeyId + Gpg Fingerprint) -> can only be IDed if GPG known.
+
+	bool mGpgIdKnown; // if GpgIdHash has been identified.
+	std::string mGpgId;   	// if known.
+	std::string mGpgName; 	// if known.
+	std::string mGpgEmail; 	// if known.
+};
+
+
+
+class RsIdMsg
+{
+	public:
+
+	RsMsgMetaData mMeta;
+
+	// In MsgMetaData.
+	//std::string mKeyId;  (mGroupId)
+	//std::string mPeerId; (mAuthorId) ???
+
+	int mRating;
+	int mPeersRating;
+	std::string mComment;
+};
+
+
+
+
+#if 0
+class RsIdReputation
+{
+	public:
+	std::string mKeyId;
+
+	int mYourRating;
+	int mPeersRating;
+	int mFofRating;
+	int mTotalRating;
+
+	std::string mComment;
+};
+
+class RsIdOpinion
+{
+	public:
+
+	std::string mKeyId;
+	std::string mPeerId;
+
+	int mRating;
+	int mPeersRating;
+	std::string mComment;
+};
+
+#endif
+
+
+class RsIdentity: public RsTokenService
+{
+	public:
+
+	RsIdentity()  { return; }
+virtual ~RsIdentity() { return; }
+
+
+	/* INCLUDES INTERFACE FROM RS TOKEN SERVICE */
+	//////////////////////////////////////////////////////////////////////////////
+
+
+        /* Specific Service Data */
+virtual bool    getGroupData(const uint32_t &token, RsIdGroup &group) = 0;
+virtual bool    getMsgData(const uint32_t &token, RsIdMsg &msg) = 0;
+
+virtual bool    createGroup(uint32_t &token, RsIdGroup &group, bool isNew) = 0;
+virtual bool    createMsg(uint32_t &token, RsIdMsg &msg, bool isNew) = 0;
+
+	/* In the Identity System - You don't access the Messages Directly.
+	 * as they represent idividuals opinions....
+	 * This is reflected in the TokenService calls returning false.
+	 *
+	 * Below is the additional interface to look at reputation.
+	 */
+
+virtual void generateDummyData() = 0;
+
+#if 0
+        /* Data Requests */
+virtual bool requestIdentityList(uint32_t &token) = 0;
+virtual bool requestIdentities(uint32_t &token, const std::list<std::string> &ids) = 0;
+virtual bool requestIdReputations(uint32_t &token, const std::list<std::string> &ids) = 0;
+virtual bool requestIdPeerOpinion(uint32_t &token, const std::string &aboutId, const std::string &peerId) = 0;
+//virtual bool requestIdGpgDetails(uint32_t &token, const std::list<std::string> &ids) = 0;
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token) = 0;
+
+        /* Retrieve Data */
+virtual bool getIdentityList(const uint32_t token, std::list<std::string> &ids) = 0;
+virtual bool getIdentity(const uint32_t token, RsIdData &data) = 0;
+virtual bool getIdReputation(const uint32_t token, RsIdReputation &reputation) = 0;
+virtual bool getIdPeerOpinion(const uint32_t token, RsIdOpinion &opinion) = 0;
+
+        /* Updates */
+virtual bool updateIdentity(RsIdData &data) = 0;
+virtual bool updateOpinion(RsIdOpinion &opinion) = 0;
+
+#endif
+
+};
+
+
+
+#endif

libretroshare/src/retroshare/rsphoto.h

@@ -2,11 +2,11 @@
 #define RETROSHARE_PHOTO_GUI_INTERFACE_H
 
 /*
- * libretroshare/src/rsiface: rsphoto.h
+ * libretroshare/src/retroshare: rsphoto.h
  *
  * RetroShare C++ Interface.
  *
- * Copyright 2008-2008 by Robert Fernie.
+ * Copyright 2008-2012 by Robert Fernie.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Library General Public
@@ -29,99 +29,151 @@
 #include <inttypes.h>
 #include <string>
 #include <list>
+#include <retroshare/rsidentity.h>
 
 /* The Main Interface Class - for information about your Peers */
 class RsPhoto;
 extern RsPhoto *rsPhoto;
 
-class RsPhotoDetails;
+/******************* NEW STUFF FOR NEW CACHE SYSTEM *********/
-class RsPhotoShowDetails;
 
-class RsPhotoShowInfo
+#define RSPHOTO_MODE_NEW	1
+#define RSPHOTO_MODE_OWN	2
+#define RSPHOTO_MODE_REMOTE	3
+
+class RsPhotoThumbnail
+{
+	public:
+		RsPhotoThumbnail()
+		:data(NULL), size(0), type("N/A") { return; }
+
+	bool deleteImage();
+	bool copyFrom(const RsPhotoThumbnail &nail);
+
+	// Holds Thumbnail image.
+	uint8_t *data;
+	int size;
+	std::string type;
+};
+
+
+/* If these flags are no set - the Photo inherits values from the Album
+ */
+
+#define RSPHOTO_FLAGS_ATTRIB_TITLE		0x0001
+#define RSPHOTO_FLAGS_ATTRIB_CAPTION		0x0002
+#define RSPHOTO_FLAGS_ATTRIB_DESC		0x0004
+#define RSPHOTO_FLAGS_ATTRIB_PHOTOGRAPHER	0x0008
+#define RSPHOTO_FLAGS_ATTRIB_WHERE		0x0010
+#define RSPHOTO_FLAGS_ATTRIB_WHEN		0x0020
+#define RSPHOTO_FLAGS_ATTRIB_OTHER		0x0040
+#define RSPHOTO_FLAGS_ATTRIB_CATEGORY		0x0080
+#define RSPHOTO_FLAGS_ATTRIB_HASHTAGS		0x0100
+#define RSPHOTO_FLAGS_ATTRIB_ORDER		0x0200
+#define RSPHOTO_FLAGS_ATTRIB_THUMBNAIL		0x0400
+#define RSPHOTO_FLAGS_ATTRIB_MODE		0x0800
+#define RSPHOTO_FLAGS_ATTRIB_AUTHOR		0x1000 // PUSH UP ORDER
+#define RSPHOTO_FLAGS_ATTRIB_PHOTO		0x2000 // PUSH UP ORDER.
+
+
+class RsPhotoPhoto
 {
 	public:
 
-	std::string photoId;
+	RsMsgMetaData mMeta;
-	std::wstring altComment;
+
-	uint32_t    deltaT; /* in 100ths of sec? */
+	RsPhotoPhoto();
+
+	// THESE ARE IN THE META DATA.
+	//std::string mAlbumId;
+	//std::string mId;
+	//std::string mTitle; // only used by Album.
+	std::string mCaption;
+	std::string mDescription;
+	std::string mPhotographer;
+	std::string mWhere;
+	std::string mWhen;
+	std::string mOther;
+	std::string mCategory;
+
+	std::string mHashTags;
+
+	uint32_t mSetFlags;
+
+	int mOrder;
+
+	RsPhotoThumbnail mThumbnail;
+
+	int mMode;
+
+	// These are not saved.
+	std::string path; // if in Mode NEW.
+	uint32_t mModFlags;
 };
-	
+
-class RsPhotoShowDetails
+class RsPhotoAlbumShare
 {
 	public:
 
-	RsPhotoShowDetails();
+	uint32_t mShareType;
-
+	std::string mShareGroupId;
-	std::string id;
+	std::string mPublishKey;
-	std::string showid;
+	uint32_t mCommentMode;
-
+	uint32_t mResizeMode;
-	std::string name;
-	std::wstring location;
-	std::wstring comment;
-	std::string date;
-	std::list<RsPhotoShowInfo> photos;
 };
 
-/* Details class */
+class RsPhotoAlbum
-class RsPhotoDetails
 {
 	public:
+	RsPhotoAlbum();
 
-	RsPhotoDetails();
+	RsGroupMetaData mMeta;
 
-	std::string id;
+	// THESE ARE IN THE META DATA.
-	std::string srcid;
+	//std::string mAlbumId;
+	//std::string mTitle; // only used by Album.
 
-	std::string hash;
+	std::string mCaption;
-	uint64_t size;
+	std::string mDescription;
+	std::string mPhotographer;
+	std::string mWhere;
+	std::string mWhen;
+	std::string mOther;
+	std::string mCategory;
 
-	std::string name; 
+	std::string mHashTags;
-	std::wstring comment;
 
-	std::string location;
+	RsPhotoThumbnail mThumbnail;
-	std::string date;
 
-	uint32_t format;
+	int mMode;
 
-	bool isAvailable;
+	std::string mPhotoPath;
-	std::string path;
+	RsPhotoAlbumShare mShareOptions;
+
+	// These aren't saved.
+	uint32_t mSetFlags;
+	uint32_t mModFlags;
 };
 
-std::ostream &operator<<(std::ostream &out, const RsPhotoShowDetails &detail);
+std::ostream &operator<<(std::ostream &out, const RsPhotoPhoto &photo);
-std::ostream &operator<<(std::ostream &out, const RsPhotoDetails &detail);
+std::ostream &operator<<(std::ostream &out, const RsPhotoAlbum &album);
 
-class RsPhoto
+
+class RsPhoto: public RsTokenService
 {
 	public:
 
 	RsPhoto()  { return; }
 virtual ~RsPhoto() { return; }
 
-	/* changed? */
+	/* Specific Service Data */
-virtual bool updated() = 0;
+virtual bool getAlbum(const uint32_t &token, RsPhotoAlbum &album) = 0;
+virtual bool getPhoto(const uint32_t &token, RsPhotoPhoto &photo) = 0;
 
-	/* access data */
+virtual bool submitAlbumDetails(uint32_t &token, RsPhotoAlbum &album, bool isNew) = 0;
-virtual bool getPhotoList(std::string id, std::list<std::string> &hashs) 	= 0;
+virtual bool submitPhoto(uint32_t &token, RsPhotoPhoto &photo, bool isNew) = 0;
-virtual bool getShowList(std::string id,  std::list<std::string> &showIds) 	= 0;
-virtual bool getShowDetails(std::string id, std::string showId, RsPhotoShowDetails &detail) = 0;
-virtual bool getPhotoDetails(std::string id, std::string photoId, RsPhotoDetails &detail) = 0;
-
-	/* add / delete */
-virtual std::string createShow(std::string name) 				= 0; 
-virtual bool deleteShow(std::string showId)					= 0;
-virtual bool addPhotoToShow(std::string showId, std::string photoId, int16_t index) = 0;
-virtual bool movePhotoInShow(std::string showId, std::string photoId, int16_t index) = 0;
-virtual bool removePhotoFromShow(std::string showId, std::string photoId) 	= 0;
-
-virtual std::string addPhoto(std::string path) = 0; /* add from file */
-virtual bool addPhoto(std::string srcId, std::string photoId) = 0; /* add from peers photos */
-virtual bool deletePhoto(std::string photoId) = 0;
-
-	/* modify properties (TODO) */
-virtual bool modifyShow(std::string showId, std::wstring name, std::wstring comment) = 0;
-virtual bool modifyPhoto(std::string photoId, std::wstring name, std::wstring comment) = 0;
-virtual bool modifyShowComment(std::string showId, std::string photoId, std::wstring comment) = 0;
 
 };
 
+
 #endif

libretroshare/src/retroshare/rsphotoV2.h

@@ -0,0 +1,277 @@
+#ifndef RSPHOTOV2_H
+#define RSPHOTOV2_H
+
+/*
+ * libretroshare/src/retroshare: rsphoto.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2008-2012 by Robert Fernie, Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+#include "rsgxsservice.h"
+
+/* The Main Interface Class - for information about your Peers */
+class RsPhoto;
+extern RsPhoto *rsPhoto;
+
+/******************* NEW STUFF FOR NEW CACHE SYSTEM *********/
+
+#define RSPHOTO_MODE_NEW	1
+#define RSPHOTO_MODE_OWN	2
+#define RSPHOTO_MODE_REMOTE	3
+
+class RsPhotoThumbnail
+{
+        public:
+                RsPhotoThumbnail()
+                :data(NULL), size(0), type("N/A") { return; }
+
+        bool deleteImage();
+        bool copyFrom(const RsPhotoThumbnail &nail);
+
+        // Holds Thumbnail image.
+        uint8_t *data;
+        int size;
+        std::string type;
+};
+
+
+/* If these flags are no set - the Photo inherits values from the Album
+ */
+
+#define RSPHOTO_FLAGS_ATTRIB_TITLE		0x0001
+#define RSPHOTO_FLAGS_ATTRIB_CAPTION		0x0002
+#define RSPHOTO_FLAGS_ATTRIB_DESC		0x0004
+#define RSPHOTO_FLAGS_ATTRIB_PHOTOGRAPHER	0x0008
+#define RSPHOTO_FLAGS_ATTRIB_WHERE		0x0010
+#define RSPHOTO_FLAGS_ATTRIB_WHEN		0x0020
+#define RSPHOTO_FLAGS_ATTRIB_OTHER		0x0040
+#define RSPHOTO_FLAGS_ATTRIB_CATEGORY		0x0080
+#define RSPHOTO_FLAGS_ATTRIB_HASHTAGS		0x0100
+#define RSPHOTO_FLAGS_ATTRIB_ORDER		0x0200
+#define RSPHOTO_FLAGS_ATTRIB_THUMBNAIL		0x0400
+#define RSPHOTO_FLAGS_ATTRIB_MODE		0x0800
+#define RSPHOTO_FLAGS_ATTRIB_AUTHOR		0x1000 // PUSH UP ORDER
+#define RSPHOTO_FLAGS_ATTRIB_PHOTO		0x2000 // PUSH UP ORDER.
+
+
+class RsPhotoPhoto
+{
+        public:
+
+        RsMsgMetaData mMeta;
+
+        RsPhotoPhoto();
+
+        // THESE ARE IN THE META DATA.
+        //std::string mAlbumId;
+        //std::string mId;
+        //std::string mTitle; // only used by Album.
+        std::string mCaption;
+        std::string mDescription;
+        std::string mPhotographer;
+        std::string mWhere;
+        std::string mWhen;
+        std::string mOther;
+        std::string mCategory;
+
+        std::string mHashTags;
+
+        uint32_t mSetFlags;
+
+        int mOrder;
+
+        RsPhotoThumbnail mThumbnail;
+
+        int mMode;
+
+        // These are not saved.
+        std::string path; // if in Mode NEW.
+        uint32_t mModFlags;
+};
+
+class RsPhotoAlbumShare
+{
+        public:
+
+        uint32_t mShareType;
+        std::string mShareGroupId;
+        std::string mPublishKey;
+        uint32_t mCommentMode;
+        uint32_t mResizeMode;
+};
+
+class RsPhotoAlbum
+{
+        public:
+        RsPhotoAlbum();
+
+        RsGroupMetaData mMeta;
+
+        // THESE ARE IN THE META DATA.
+        //std::string mAlbumId;
+        //std::string mTitle; // only used by Album.
+
+        std::string mCaption;
+        std::string mDescription;
+        std::string mPhotographer;
+        std::string mWhere;
+        std::string mWhen;
+        std::string mOther;
+        std::string mCategory;
+
+        std::string mHashTags;
+
+        RsPhotoThumbnail mThumbnail;
+
+        int mMode;
+
+        std::string mPhotoPath;
+        RsPhotoAlbumShare mShareOptions;
+
+        // These aren't saved.
+        uint32_t mSetFlags;
+        uint32_t mModFlags;
+};
+
+std::ostream &operator<<(std::ostream &out, const RsPhotoPhoto &photo);
+std::ostream &operator<<(std::ostream &out, const RsPhotoAlbum &album);
+
+typedef std::map<RsGxsGroupId, std::vector<RsPhotoPhoto> > PhotoResult;
+typedef std::map<RsGxsGroupId, std::vector<RsMsgMetaData> > MsgMetaResult;
+
+class RsPhotoV2
+{
+
+public:
+
+
+	RsPhotoV2()  { return; }
+
+	virtual ~RsPhotoV2() { return; }
+
+	/*!
+	 * Use to enquire if groups or msgs have changed
+	 * Poll regularly, particularly after a photo submission
+	 * @return true if msgs or groups have changed
+	 */
+	virtual bool updated() = 0;
+
+	/*!
+	 *
+	 * @param grpIds
+	 */
+	virtual void groupsChanged(std::list<RsGroupId>& grpIds) = 0;
+
+	/*!
+	 *
+	 * @param msgs
+	 */
+	virtual void msgsChanged(GxsMsgIdResult& msgs) = 0;
+
+	/*!
+	 * To acquire a handle to token service handler
+	 * needed to make requests to the service
+	 * @return handle to token service for this gxs service
+	 */
+	virtual RsTokenService* getTokenService() = 0;
+
+	/* Generic Lists */
+
+	/*!
+	 *
+	 * @param token token to be redeemed for this request
+	 * @param groupIds the ids return for given request token
+	 * @return false if request token is invalid, check token status for error report
+	 */
+	virtual bool getGroupList(const uint32_t &token,
+							  std::list<RsGroupId> &groupIds) = 0;
+
+	/*!
+	 * @param token token to be redeemed for this request
+	 * @param msgIds the ids return for given request token
+	 * @return false if request token is invalid, check token status for error report
+	 */
+	virtual bool getMsgList(const uint32_t &token,
+							GxsMsgIdResult &msgIds) = 0;
+
+	/* Generic Summary */
+
+	/*!
+	 * @param token token to be redeemed for group summary request
+	 * @param groupInfo the ids returned for given request token
+	 * @return false if request token is invalid, check token status for error report
+	 */
+	virtual bool getGroupSummary(const uint32_t &token,
+								 std::list<RsGroupMetaData> &groupInfo) = 0;
+
+	/*!
+	 * @param token token to be redeemed for message summary request
+	 * @param msgInfo the message metadata returned for given request token
+	 * @return false if request token is invalid, check token status for error report
+	 */
+	virtual bool getMsgSummary(const uint32_t &token,
+							   MsgMetaResult &msgInfo) = 0;
+
+			/* Specific Service Data */
+
+	/*!
+	 * @param token token to be redeemed for album request
+	 * @param album the album returned for given request token
+	 * @return false if request token is invalid, check token status for error report
+	 */
+	virtual bool getAlbum(const uint32_t &token, std::vector<RsPhotoAlbum> &album) = 0;
+
+	/*!
+	 * @param token token to be redeemed for photo request
+	 * @param photo the photo returned for given request token
+	 * @return false if request token is invalid, check token status for error report
+	 */
+	virtual bool getPhoto(const uint32_t &token,
+						  PhotoResult &photo) = 0;
+
+	/* details are updated in album - to choose Album ID, and storage path */
+
+	/*!
+	 * This RsGenExchange service will be alerted to this album as \n
+	 * a new album. Do not keep the submitted album as representative, wait for
+	 * notification telling of successful submission
+	 * @param album The album to be submitted
+	 * @return false if submission failed
+	 */
+	virtual bool submitAlbumDetails(RsPhotoAlbum &album) = 0;
+
+	/*!
+	 * This RsGenExchange service will be alerted to this photo as \n
+	 * a new photo. Do not keep the submitted photo as representative, wait for new photo
+	 * returned
+	 * @param photo photo to submit
+	 * @return
+	 */
+	virtual bool submitPhoto(RsPhotoPhoto &photo) = 0;
+
+};
+
+
+#endif // RSPHOTOV2_H

libretroshare/src/retroshare/rsposted.h

@@ -0,0 +1,140 @@
+#ifndef RETROSHARE_POSTED_GUI_INTERFACE_H
+#define RETROSHARE_POSTED_GUI_INTERFACE_H
+
+/*
+ * libretroshare/src/retroshare: rsposted.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2008-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+#include <retroshare/rsidentity.h>
+
+/* The Main Interface Class - for information about your Peers */
+class RsPosted;
+extern RsPosted *rsPosted;
+
+
+class RsPostedGroup
+{
+	public:
+	RsGroupMetaData mMeta;
+	RsPostedGroup() { return; }
+};
+
+class RsPostedMsg
+{
+	public:
+	RsPostedMsg(uint32_t t)
+	:postedType(t) { return; }
+
+	RsMsgMetaData mMeta;
+	uint32_t postedType;
+};
+
+#define RSPOSTED_MSGTYPE_POST		0x0001
+#define RSPOSTED_MSGTYPE_VOTE		0x0002
+#define RSPOSTED_MSGTYPE_COMMENT	0x0004
+
+#define RSPOSTED_PERIOD_YEAR		1
+#define RSPOSTED_PERIOD_MONTH		2
+#define RSPOSTED_PERIOD_WEEK		3
+#define RSPOSTED_PERIOD_DAY		4
+#define RSPOSTED_PERIOD_HOUR		5
+
+#define RSPOSTED_VIEWMODE_LATEST	1
+#define RSPOSTED_VIEWMODE_TOP		2
+#define RSPOSTED_VIEWMODE_HOT		3
+#define RSPOSTED_VIEWMODE_COMMENTS	4
+
+
+class RsPostedPost: public RsPostedMsg
+{
+	public:
+	RsPostedPost(): RsPostedMsg(RSPOSTED_MSGTYPE_POST) 
+	{ 
+		mMeta.mMsgFlags = RSPOSTED_MSGTYPE_POST;
+		return; 
+	}
+};
+
+
+class RsPostedVote: public RsPostedMsg
+{
+	public:
+	RsPostedVote(): RsPostedMsg(RSPOSTED_MSGTYPE_VOTE)
+	{ 
+		mMeta.mMsgFlags = RSPOSTED_MSGTYPE_VOTE;
+		return; 
+	}
+};
+
+
+class RsPostedComment: public RsPostedMsg
+{
+	public:
+	RsPostedComment(): RsPostedMsg(RSPOSTED_MSGTYPE_COMMENT) 
+	{ 
+		mMeta.mMsgFlags = RSPOSTED_MSGTYPE_COMMENT;
+		return; 
+	}
+};
+
+
+std::ostream &operator<<(std::ostream &out, const RsPostedGroup &group);
+std::ostream &operator<<(std::ostream &out, const RsPostedPost &post);
+std::ostream &operator<<(std::ostream &out, const RsPostedVote &vote);
+std::ostream &operator<<(std::ostream &out, const RsPostedComment &comment);
+
+
+class RsPosted: public RsTokenService
+{
+	public:
+
+	RsPosted()  { return; }
+virtual ~RsPosted() { return; }
+
+	/* Specific Service Data */
+virtual bool getGroup(const uint32_t &token, RsPostedGroup &group) = 0;
+virtual bool getPost(const uint32_t &token, RsPostedPost &post) = 0;
+virtual bool getComment(const uint32_t &token, RsPostedComment &comment) = 0;
+
+virtual bool submitGroup(uint32_t &token, RsPostedGroup &group, bool isNew) = 0;
+virtual bool submitPost(uint32_t &token, RsPostedPost &post, bool isNew) = 0;
+virtual bool submitVote(uint32_t &token, RsPostedVote &vote, bool isNew) = 0;
+virtual bool submitComment(uint32_t &token, RsPostedComment &comment, bool isNew) = 0;
+
+	// Special Ranking Request.
+virtual bool requestRanking(uint32_t &token, std::string groupId) = 0;
+virtual bool getRankedPost(const uint32_t &token, RsPostedPost &post) = 0;
+
+virtual bool extractPostedCache(const std::string &str, uint32_t &votes, uint32_t &comments) = 0;
+
+	// exposed for testing...
+virtual float calcPostScore(const RsMsgMetaData &meta) = 0;
+
+};
+
+
+#endif

libretroshare/src/retroshare/rswiki.h

@@ -0,0 +1,106 @@
+#ifndef RETROSHARE_WIKI_GUI_INTERFACE_H
+#define RETROSHARE_WIKI_GUI_INTERFACE_H
+
+/*
+ * libretroshare/src/retroshare: rswiki.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+
+#include <retroshare/rsidentity.h>
+
+/* The Main Interface Class - for information about your Peers */
+class RsWiki;
+extern RsWiki *rsWiki;
+
+class RsWikiGroupShare
+{
+	public:
+
+	uint32_t mShareType;
+	std::string mShareGroupId;
+	std::string mPublishKey;
+	uint32_t mCommentMode;
+	uint32_t mResizeMode;
+};
+
+class RsWikiGroup
+{
+	public:
+
+	RsGroupMetaData mMeta;
+
+	//std::string mGroupId;
+	//std::string mName;
+
+	std::string mDescription;
+	std::string mCategory;
+
+	std::string mHashTags;
+
+	RsWikiGroupShare mShareOptions;
+};
+
+class RsWikiPage
+{
+	public:
+
+	RsMsgMetaData mMeta;
+
+	// IN META DATA.
+	//std::string mGroupId;
+	//std::string mOrigPageId;
+	//std::string mPageId;
+	//std::string mName;
+
+	// WE SHOULD SWITCH TO USING THREAD/PARENT IDS HERE....
+	std::string mPrevId;
+
+	std::string mPage; // all the text is stored here.
+
+	std::string mHashTags;
+};
+
+class RsWiki: public RsTokenService
+{
+	public:
+
+	RsWiki()  { return; }
+virtual ~RsWiki() { return; }
+
+	/* Specific Service Data */
+virtual bool getGroupData(const uint32_t &token, RsWikiGroup &group) = 0;
+virtual bool getMsgData(const uint32_t &token, RsWikiPage &page) = 0;
+
+virtual bool createGroup(uint32_t &token, RsWikiGroup &group, bool isNew) = 0;
+virtual bool createPage(uint32_t &token, RsWikiPage &page, bool isNew) = 0;
+
+
+};
+
+
+
+#endif

libretroshare/src/retroshare/rswire.h

@@ -0,0 +1,103 @@
+#ifndef RETROSHARE_WIRE_GUI_INTERFACE_H
+#define RETROSHARE_WIRE_GUI_INTERFACE_H
+
+/*
+ * libretroshare/src/retroshare: rswire.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <inttypes.h>
+#include <string>
+#include <list>
+
+#include <retroshare/rsidentity.h>
+
+/* The Main Interface Class - for information about your Peers */
+class RsWire;
+extern RsWire *rsWire;
+
+class RsWireGroupShare
+{
+	public:
+
+	uint32_t mShareType;
+	std::string mShareGroupId;
+	std::string mPublishKey;
+	uint32_t mCommentMode;
+	uint32_t mResizeMode;
+};
+
+class RsWireGroup
+{
+	public:
+
+	RsGroupMetaData mMeta;
+
+	//std::string mGroupId;
+	//std::string mName;
+
+	std::string mDescription;
+	std::string mCategory;
+
+	std::string mHashTags;
+
+	RsWireGroupShare mShareOptions;
+};
+
+class RsWirePulse
+{
+	public:
+
+	RsMsgMetaData mMeta;
+
+	//std::string mGroupId;
+	//std::string mOrigPageId;
+	//std::string mPrevId;
+	//std::string mPageId;
+	//std::string mName;
+
+	std::string mPulse; // all the text is stored here.
+
+	std::string mHashTags;
+};
+
+class RsWire: public RsTokenService
+{
+	public:
+
+	RsWire()  { return; }
+virtual ~RsWire() { return; }
+
+	/* Specific Service Data */
+virtual bool getGroupData(const uint32_t &token, RsWireGroup &group) = 0;
+virtual bool getMsgData(const uint32_t &token, RsWirePulse &pulse) = 0;
+
+	/* Create Stuff */
+virtual bool createGroup(uint32_t &token, RsWireGroup &group, bool isNew) = 0;
+virtual bool createPulse(uint32_t &token, RsWirePulse &pulse, bool isNew) = 0;
+
+};
+
+
+
+#endif

libretroshare/src/rsserver/rsinit.cc

@@ -1019,7 +1019,7 @@ std::string RsInit::getRetroshareDataDirectory()
 static bool isHexaString(const std::string& s)
 {
 	for(uint32_t i=0;i<s.length();++i)
-		if(!(s[i] >= 'A' && s[i] <= 'F' || s[i] >= '0' && s[i] <= '9' || s[i] >= 'a' && s[i] <= 'f'))
+		if(!((s[i] >= 'A' && s[i] <= 'F') || (s[i] >= '0' && s[i] <= '9') || (s[i] >= 'a' && s[i] <= 'f')))
 			return false ;
 
 	return true ;
@@ -1800,13 +1800,21 @@ RsTurtle *rsTurtle = NULL ;
 #include "services/p3msgservice.h"
 #include "services/p3chatservice.h"
 #include "services/p3gamelauncher.h"
-#include "services/p3photoservice.h"
 #include "services/p3forums.h"
 #include "services/p3channels.h"
 #include "services/p3statusservice.h"
 #include "services/p3blogs.h"
 #include "turtle/p3turtle.h"
 
+#ifdef ENABLE_GXS_SERVICES
+#include "services/p3photoservice.h"
+#include "services/p3wikiservice.h"
+#include "services/p3wire.h"
+#include "services/p3idservice.h"
+#include "services/p3forumsv2.h"
+#include "services/p3posted.h"
+#endif
+
 #ifndef PQI_DISABLE_TUNNEL
 #include "services/p3tunnel.h"
 #endif
@@ -1823,7 +1831,6 @@ RsTurtle *rsTurtle = NULL ;
 #include "rsserver/p3peers.h"
 #include "rsserver/p3msgs.h"
 #include "rsserver/p3discovery.h"
-#include "rsserver/p3photo.h"
 #include "rsserver/p3status.h"
 #include "rsserver/p3history.h"
 #include "rsserver/p3serverconfig.h"
@@ -2248,16 +2255,36 @@ int RsServer::StartupRetroShare()
 	mPluginsManager->registerClientServices(pqih) ;
 	mPluginsManager->registerCacheServices() ;
 
+#ifdef ENABLE_GXS_SERVICES
+	// Testing New Cache Services.
+	p3PhotoService *mPhotos = new p3PhotoService(RS_SERVICE_TYPE_PHOTO);
+	pqih -> addService(mPhotos);
+	
+	// Testing New Cache Services.
+	p3WikiService *mWikis = new p3WikiService(RS_SERVICE_TYPE_WIKI);
+	pqih -> addService(mWikis);
+	
+	// Testing New Cache Services.
+	p3Wire *mWire = new p3Wire(RS_SERVICE_TYPE_WIRE);
+	pqih -> addService(mWire);
+	
+	// Testing New Cache Services.
+	p3IdService *mIdentity = new p3IdService(RS_SERVICE_TYPE_IDENTITY);
+	pqih -> addService(mIdentity);
+	
+	// Testing New Cache Services.
+	p3ForumsV2 *mForumsV2 = new p3ForumsV2(RS_SERVICE_TYPE_FORUMSV2);
+	pqih -> addService(mForumsV2);
+	
+	// Testing New Cache Services.
+	p3PostedService *mPosted = new p3PostedService(RS_SERVICE_TYPE_POSTED);
+	pqih -> addService(mPosted);
+#endif // ENABLE_GXS_SERVICES
+
+
 #ifndef RS_RELEASE
 	p3GameLauncher *gameLauncher = new p3GameLauncher(mLinkMgr);
 	pqih -> addService(gameLauncher);
-
-	p3PhotoService *photoService = new p3PhotoService(RS_SERVICE_TYPE_PHOTO,   /* .... for photo service */
-			mCacheStrapper, mCacheTransfer,
-			localcachedir, remotecachedir);
-
-        CachePair cp2(photoService, photoService, CacheId(RS_SERVICE_TYPE_PHOTO, 0));
-	mCacheStrapper -> addCachePair(cp2);
 #endif
 
 #ifdef RS_VOIPTEST
@@ -2510,6 +2537,18 @@ int RsServer::StartupRetroShare()
 	rsMsgs  = new p3Msgs(msgSrv, chatSrv);
 	rsForums = mForums;
 	rsChannels = mChannels;
+
+#ifdef ENABLE_GXS_SERVICES
+	// Testing of new cache system interfaces.
+	rsIdentity = mIdentity;
+	rsPhoto = mPhotos;
+	rsWiki = mWikis;
+	rsWire = mWire;
+	rsForumsV2 = mForumsV2;
+	rsPosted = mPosted;
+#endif // ENABLE_GXS_SERVICES
+
+
 #ifdef RS_USE_BLOGS	
 	rsBlogs = mBlogs;
 #endif
@@ -2518,10 +2557,8 @@ int RsServer::StartupRetroShare()
 
 #ifndef RS_RELEASE
 	rsGameLauncher = gameLauncher;
-	rsPhoto = new p3Photo(photoService);
 #else
 	rsGameLauncher = NULL;
-	rsPhoto = NULL;
 #endif
 
 

libretroshare/src/serialiser/rsgxsitem.h

@@ -0,0 +1,27 @@
+#ifndef RSGXSITEM_H
+#define RSGXSITEM_H
+
+#include "serialiser/rsserviceids.h"
+#include "serialiser/rsserial.h"
+#include "serialiser/rstlvbase.h"
+#include "serialiser/rstlvtypes.h"
+#include "serialiser/rstlvkeys.h"
+
+
+class RsGxsGrpItem : RsItem
+{
+
+    RsGxsItem() : RsItem(0) { return; }
+    virtual ~RsGxsItem();
+
+};
+
+class RsGxsMsgItem : RsItem
+{
+
+    RsGxsItem() : RsItem(0) { return; }
+    virtual ~RsGxsItem();
+
+};
+
+#endif // RSGXSITEM_H

libretroshare/src/serialiser/rsgxsitems.cc

@@ -0,0 +1,40 @@
+/*
+ * rsgxsitems.cc
+ *
+ *  Created on: 26 Jul 2012
+ *      Author: crispy
+ */
+
+
+#include "rsgxsitems.h"
+#include "gxs/rsgxsdata.h"
+
+	void RsMsgMetaData::operator =(const RsGxsMsgMetaData& rGxsMeta)
+	{
+		this->mAuthorId = rGxsMeta.mAuthorId;
+		this->mChildTs = rGxsMeta.mChildTs;
+		this->mGroupId = rGxsMeta.mGroupId;
+		this->mMsgFlags = rGxsMeta.mMsgFlags;
+		this->mMsgId = rGxsMeta.mMsgId;
+		this->mMsgName = rGxsMeta.mMsgName;
+		this->mMsgStatus = rGxsMeta.mMsgStatus;
+		this->mOrigMsgId = rGxsMeta.mOrigMsgId;
+		this->mParentId = rGxsMeta.mParentId;
+		this->mPublishTs = rGxsMeta.mPublishTs;
+		this->mThreadId = rGxsMeta.mThreadId;
+	}
+
+
+	void RsGroupMetaData::operator =(const RsGxsGrpMetaData& rGxsMeta)
+	{
+		this->mAuthorId = rGxsMeta.mAuthorId;
+		this->mGroupFlags = rGxsMeta.mGroupFlags;
+		this->mGroupId = rGxsMeta.mGroupId;
+		this->mGroupStatus = rGxsMeta.mGroupStatus;
+		this->mLastPost = rGxsMeta.mLastPost;
+		this->mMsgCount = rGxsMeta.mMsgCount;
+		this->mPop = rGxsMeta.mPop;
+		this->mPublishTs = rGxsMeta.mPublishTs;
+		this->mSubscribeFlags = rGxsMeta.mSubscribeFlags;
+		this->mGroupName = rGxsMeta.mGroupName;
+	}

libretroshare/src/serialiser/rsgxsitems.h

@@ -0,0 +1,155 @@
+#ifndef RSGXSITEMS_H
+#define RSGXSITEMS_H
+
+/*
+ * libretroshare/src/serialiser: rsgxsitems.h
+ *
+ * RetroShare Serialiser.
+ *
+ * Copyright 2012   Christopher Evi-Parker, Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "serialiser/rsserviceids.h"
+#include "serialiser/rsserial.h"
+#include "serialiser/rstlvbase.h"
+#include "serialiser/rstlvtypes.h"
+#include "serialiser/rstlvkeys.h"
+
+class RsGxsGrpMetaData;
+class RsGxsMsgMetaData;
+
+class RsGroupMetaData
+{
+        public:
+
+        RsGroupMetaData()
+        {
+                mGroupFlags = 0;
+                mSubscribeFlags = 0;
+
+                mPop = 0;
+                mMsgCount = 0;
+                mLastPost = 0;
+                mGroupStatus = 0;
+
+                //mPublishTs = 0;
+        }
+
+    	void operator =(const RsGxsGrpMetaData& rGxsMeta);
+//    	{
+//    		this->mAuthorId = rGxsMeta.mAuthorId;
+//    		this->mGroupFlags = rGxsMeta.mGroupFlags;
+//    		this->mGroupId = rGxsMeta.mGroupId;
+//    		this->mGroupStatus = rGxsMeta.mGroupStatus;
+//    		this->mLastPost = rGxsMeta.mLastPost;
+//    		this->mMsgCount = rGxsMeta.mMsgCount;
+//    		this->mPop = rGxsMeta.mPop;
+//    		this->mPublishTs = rGxsMeta.mPublishTs;
+//    		this->mSubscribeFlags = rGxsMeta.mSubscribeFlags;
+//    		this->mGroupName = rGxsMeta.mGroupName;
+//    	}
+
+        std::string mGroupId;
+        std::string mGroupName;
+        uint32_t    mGroupFlags;
+
+        time_t      mPublishTs; // Mandatory.
+        std::string mAuthorId;   // Optional.
+
+        // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.
+
+        uint32_t    mSubscribeFlags;
+
+        uint32_t    mPop; // HOW DO WE DO THIS NOW.
+        uint32_t    mMsgCount; // ???
+        time_t      mLastPost; // ???
+
+        uint32_t    mGroupStatus;
+
+};
+
+
+
+
+class RsMsgMetaData
+{
+        public:
+
+        RsMsgMetaData()
+        {
+                mPublishTs = 0;
+                mMsgFlags = 0;
+                mMsgStatus = 0;
+                mChildTs = 0;
+        }
+
+        void operator =(const RsGxsMsgMetaData& rGxsMeta);
+
+
+        std::string mGroupId;
+        std::string mMsgId;
+
+        std::string mThreadId;
+        std::string mParentId;
+        std::string mOrigMsgId;
+
+        std::string mAuthorId;
+
+        std::string mMsgName;
+        time_t      mPublishTs;
+
+        uint32_t    mMsgFlags; // Whats this for?
+
+        // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.
+        // normally READ / UNREAD flags. LOCAL Data.
+        uint32_t    mMsgStatus;
+        time_t      mChildTs;
+
+};
+
+
+class RsGxsGrpItem : public RsItem
+{
+
+public:
+
+    RsGxsGrpItem() : RsItem(0) { return; }
+    virtual ~RsGxsGrpItem(){}
+
+    RsGroupMetaData meta;
+
+};
+
+class RsGxsMsgItem : public RsItem
+{
+
+public:
+    RsGxsMsgItem() : RsItem(0) { return; }
+    virtual ~RsGxsMsgItem(){}
+
+    RsMsgMetaData meta;
+
+};
+
+
+
+
+
+#endif // RSGXSITEMS_H

libretroshare/src/serialiser/rsnxsitems.h

@@ -0,0 +1,462 @@
+#ifndef RSNXSITEMS_H
+#define RSNXSITEMS_H
+
+/*
+ * libretroshare/src/serialiser: rsnxssitems.h
+ *
+ * RetroShare Serialiser.
+ *
+ * Copyright 2012 Christopher Evi-Parker, Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+
+#include <map>
+
+#include "serialiser/rsserviceids.h"
+#include "serialiser/rsserial.h"
+#include "serialiser/rstlvbase.h"
+#include "serialiser/rstlvtypes.h"
+#include "serialiser/rstlvkeys.h"
+
+#include "gxs/rsgxsdata.h"
+
+
+const uint8_t RS_PKT_SUBTYPE_NXS_SYNC_GRP      = 0x0001;
+const uint8_t RS_PKT_SUBTYPE_NXS_SYNC_GRP_ITEM = 0x0002;
+const uint8_t RS_PKT_SUBTYPE_NXS_GRP     = 0x0004;
+const uint8_t RS_PKT_SUBTYPE_NXS_SYNC_MSG_ITEM      = 0x0008;
+const uint8_t RS_PKT_SUBTYPE_NXS_SYNC_MSG = 0x0010;
+const uint8_t RS_PKT_SUBTYPE_NXS_MSG      = 0x0020;
+const uint8_t RS_PKT_SUBTYPE_NXS_TRANS      = 0x0040;
+
+
+// possibility create second service to deal with this functionality
+
+const uint8_t RS_PKT_SUBTYPE_NXS_EXTENDED      = 0x0080; // in order to extend supported pkt subtypes
+const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_GRP   = 0x0001;
+const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_MSG   = 0x0002;
+const uint8_t RS_PKT_SUBTYPE_EXT_DELETE_GRP   = 0x0004;
+const uint8_t RS_PKT_SUBTYPE_EXT_DELETE_MSG   = 0x0008;
+const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_REQ    = 0x0010;
+
+
+/*!
+ * Base class for Network exchange service
+ * Main purpose is for rtti based routing used in the
+ * serialisation and deserialisation of NXS packets
+ *
+ * Service type is set by plugin service
+ */
+class RsNxsItem : public RsItem
+{
+
+public:
+    RsNxsItem(uint16_t servtype, uint8_t subtype)
+        : RsItem(RS_PKT_VERSION_SERVICE, servtype, subtype), transactionNumber(0) { return; }
+
+    virtual void clear() = 0;
+    virtual std::ostream &print(std::ostream &out, uint16_t indent = 0) = 0;
+
+    uint32_t transactionNumber; // set to zero if this is not a transaction item
+};
+
+
+/*!
+ * Use to request grp list from peer
+ * Server may advise client peer to use sync file
+ * while serving his request. This results
+ */
+class RsNxsSyncGrp : public RsNxsItem {
+
+public:
+
+    static const uint8_t FLAG_USE_SYNC_HASH;
+    static const uint8_t FLAG_ONLY_CURRENT; // only send most current version of grps / ignores sync hash
+
+    RsNxsSyncGrp(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_SYNC_GRP) { clear(); return;}
+
+    virtual void clear();
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    uint8_t flag; // advises whether to use sync hash
+    uint32_t syncAge; // how far back to sync data
+    std::string syncHash; // use to determine if changes that have occured since last hash
+
+
+};
+
+
+
+/*!
+ * This RsNxsItem is for use in enabling transactions
+ * in order to guaranttee a collection of item have been
+ * received
+ */
+class RsNxsTransac : public RsNxsItem {
+
+public:
+
+    static const uint16_t FLAG_STATE_MASK = 0xff;
+    static const uint16_t FLAG_TYPE_MASK = 0xff00;
+
+    /** transaction state **/
+    static const uint16_t FLAG_BEGIN_P1;
+    static const uint16_t FLAG_BEGIN_P2;
+    static const uint16_t FLAG_END_SUCCESS;
+    static const uint16_t FLAG_CANCEL;
+    static const uint16_t FLAG_END_FAIL_NUM;
+    static const uint16_t FLAG_END_FAIL_TIMEOUT;
+    static const uint16_t FLAG_END_FAIL_FULL;
+
+
+    /** transaction type **/
+    static const uint16_t FLAG_TYPE_GRP_LIST_RESP;
+    static const uint16_t FLAG_TYPE_MSG_LIST_RESP;
+    static const uint16_t FLAG_TYPE_GRP_LIST_REQ;
+    static const uint16_t FLAG_TYPE_MSG_LIST_REQ;
+    static const uint16_t FLAG_TYPE_GRPS;
+    static const uint16_t FLAG_TYPE_MSGS;
+
+    RsNxsTransac(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_TRANS) { clear(); return; }
+    virtual ~RsNxsTransac() { return ; }
+
+    virtual void clear();
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    uint16_t transactFlag;
+    uint32_t nItems;
+    uint32_t timestamp;
+};
+
+/*!
+ * Use to send to peer list of grps
+ * held by server peer
+ */
+class RsNxsSyncGrpItem : public RsNxsItem
+{
+
+public:
+
+    static const uint8_t FLAG_REQUEST;
+    static const uint8_t FLAG_RESPONSE;
+    static const uint8_t FLAG_USE_SYNC_HASH;
+
+    RsNxsSyncGrpItem(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_SYNC_GRP_ITEM) { clear(); return ; }
+    virtual ~RsNxsSyncGrpItem() { return; }
+
+    virtual void clear();
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+
+    uint8_t flag; // request or response
+
+    uint32_t publishTs; // to compare to Ts of receiving peer's grp of same id
+
+    /// grpId of grp held by sending peer
+    std::string grpId;
+
+};
+
+/*!
+ * Contains serialised group items
+ * Each item corresponds to a group which needs to be
+ * deserialised
+ */
+class RsNxsGrp : public RsNxsItem
+{
+
+public:
+
+    RsNxsGrp(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_GRP), grp(servtype), meta(servtype),
+    metaData(NULL) { clear(); return; }
+    virtual ~RsNxsGrp() { if(metaData) delete metaData; }
+
+    virtual void clear();
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    std::string grpId; /// group Id, needed to complete version Id (ncvi)
+
+    RsTlvBinaryData grp; /// actual group data
+
+    /*!
+     * This should contains all the data
+     * which is not specific to the Gxs service data
+     */
+    RsTlvBinaryData meta;
+
+    // deserialised metaData, this is not serialised
+    RsGxsGrpMetaData* metaData;
+};
+
+/*!
+ * Use to request list of msg held by peer
+ * for a given group
+ */
+class RsNxsSyncMsg : public RsNxsItem
+{
+
+public:
+
+    static const uint8_t FLAG_USE_SYNC_HASH;
+
+    RsNxsSyncMsg(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_SYNC_MSG) { clear(); return; }
+
+
+    virtual void clear();
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    std::string grpId;
+    uint8_t flag;
+    uint32_t syncAge;
+    std::string syncHash;
+};
+
+/*!
+ * Use to send list msgs for a group held by
+ * a peer
+ */
+class RsNxsSyncMsgItem : public RsNxsItem
+{
+public:
+
+    static const uint8_t FLAG_REQUEST;
+    static const uint8_t FLAG_RESPONSE;
+    static const uint8_t FLAG_USE_SYNC_HASH;
+    RsNxsSyncMsgItem(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_SYNC_MSG_ITEM) { clear(); return; }
+
+    virtual void clear();
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    uint8_t flag; // response/req
+    std::string grpId;
+    std::string msgId;
+
+};
+
+
+/*!
+ * Used to respond to a RsGrpMsgsReq
+ * with message items satisfying request
+ */
+class RsNxsMsg : public RsNxsItem
+{
+public:
+
+    RsNxsMsg(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_MSG), msg(servtype), meta(servtype) { clear(); return; }
+
+    virtual void clear();
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    std::string grpId; /// group id, forms part of version id
+    std::string msgId; /// msg id
+
+    /*!
+     * This should contains all the data
+     * which is not specific to the Gxs service data
+     */
+    RsTlvBinaryData meta;
+
+    /*!
+     * This contains Gxs specific data
+     * only client of API knows who to decode this
+     */
+    RsTlvBinaryData msg;
+
+    RsGxsMsgMetaData* metaData;
+
+};
+
+/*!
+ * Used to request a search of user data
+ */
+class RsNxsSearchReq : public RsNxsItem
+{
+public:
+
+    RsNxsSearchReq(uint16_t servtype): RsNxsItem(servtype, RS_PKT_SUBTYPE_EXT_SEARCH_REQ), serviceSearchItem(servtype) { return; }
+    virtual ~RsNxsSearchReq() { return;}
+
+    virtual void clear() { return;}
+    virtual std::ostream &print(std::ostream &out, uint16_t indent) { return out; }
+
+    uint8_t nHops; /// how many peers to jump to
+    uint32_t token; // search token
+    RsTlvBinaryData serviceSearchItem; // service aware of item class
+    uint32_t expiration; // expiration date
+};
+
+
+/*!
+ * used to extend data types processed
+ */
+class RsNxsExtended : public RsNxsItem
+{
+
+public:
+
+    RsNxsExtended(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_EXTENDED), extData(servtype) { return; }
+    virtual ~RsNxsExtended() { return; }
+
+    virtual void clear() {}
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    RsTlvBinaryData extData;
+    uint32_t type;
+};
+
+
+/*!
+ * Used to respond to a RsGrpSearchReq
+ * with grpId/MsgIds that satisfy search request
+ */
+class RsNxsSearchResultMsg
+{
+public:
+
+    RsNxsSearchResultMsg() : context(0) { return;}
+    void clear() {}
+    std::ostream &print(std::ostream &out, uint16_t indent) { return out; }
+
+    uint32_t token; // search token to be redeemed
+    RsTlvBinaryData context; // used by client service
+    std::string msgId;
+    std::string grpId;
+    RsTlvKeySignature idSign;
+
+    uint32_t expiration; // expiration date
+};
+
+/*!
+ * Used to respond to a RsGrpSearchReq
+ * with grpId/MsgIds that satisfy search request
+ */
+class RsNxsSearchResultGrp
+{
+public:
+
+    RsNxsSearchResultGrp();
+    void clear() {}
+    std::ostream &print(std::ostream &out, uint16_t indent) { return out; }
+
+    uint32_t token; // search token to be redeemed
+
+    RsTlvBinaryData context; // used by client service
+
+    std::string grpId;
+    RsTlvKeySignature adminSign;
+
+    uint32_t expiration; // expiration date
+};
+
+
+class RsNxsDeleteMsg
+{
+public:
+
+    RsNxsDeleteMsg() { return; }
+
+    std::string msgId;
+    std::string grpId;
+    RsTlvKeySignature deleteSign; // ( msgId + grpId + msg data ) sign //TODO: add warning not to place msgId+grpId in msg!
+
+};
+
+class RsNxsDeleteGrp
+{
+public:
+
+    RsNxsDeleteGrp() { return;}
+
+    std::string grpId;
+    RsTlvKeySignature idSign;
+    RsTlvKeySignature deleteSign; // (grpId + grp data) sign // TODO: add warning not to place grpId in msg
+};
+
+
+
+class RsNxsSerialiser : public RsSerialType
+{
+public:
+
+    RsNxsSerialiser(uint16_t servtype) :
+            RsSerialType(RS_PKT_VERSION_SERVICE, servtype), SERVICE_TYPE(servtype) { return; }
+
+    virtual ~RsNxsSerialiser() { return; }
+
+    virtual uint32_t size(RsItem *item);
+    virtual bool serialise(RsItem *item, void *data, uint32_t *size);
+    virtual RsItem* deserialise(void *data, uint32_t *size);
+
+private:
+
+
+    /* for RS_PKT_SUBTYPE_SYNC_GRP */
+
+    virtual uint32_t sizeNxsSyncGrp(RsNxsSyncGrp* item);
+    virtual bool serialiseNxsSyncGrp(RsNxsSyncGrp *item, void *data, uint32_t *size);
+    virtual RsNxsSyncGrp* deserialNxsSyncGrp(void *data, uint32_t *size);
+
+    /* for RS_PKT_SUBTYPE_SYNC_GRP_ITEM */
+
+    virtual uint32_t sizeNxsSyncGrpItem(RsNxsSyncGrpItem* item);
+    virtual bool serialiseNxsSyncGrpItem(RsNxsSyncGrpItem *item, void *data, uint32_t *size);
+    virtual RsNxsSyncGrpItem* deserialNxsSyncGrpItem(void *data, uint32_t *size);
+
+    /* for RS_PKT_SUBTYPE_NXS_GRP */
+
+    virtual uint32_t sizeNxsGrp(RsNxsGrp* item);
+    virtual bool serialiseNxsGrp(RsNxsGrp *item, void *data, uint32_t *size);
+    virtual RsNxsGrp* deserialNxsGrp(void *data, uint32_t *size);
+
+    /* for RS_PKT_SUBTYPE_SYNC_MSG */
+
+    virtual uint32_t sizeNxsSyncMsg(RsNxsSyncMsg* item);
+    virtual bool serialiseNxsSyncMsg(RsNxsSyncMsg *item, void *data, uint32_t *size);
+    virtual RsNxsSyncMsg* deserialNxsSyncMsg(void *data, uint32_t *size);
+
+    /* RS_PKT_SUBTYPE_SYNC_MSG_ITEM */
+
+    virtual uint32_t sizeNxsSyncMsgItem(RsNxsSyncMsgItem* item);
+    virtual bool serialiseNxsSynMsgItem(RsNxsSyncMsgItem* item, void *data, uint32_t* size);
+    virtual RsNxsSyncMsgItem* deserialNxsSyncMsgItem(void *data, uint32_t *size);
+
+    /* RS_PKT_SUBTYPE_NXS_MSG */
+
+    virtual uint32_t sizeNxsMsg(RsNxsMsg* item);
+    virtual bool serialiseNxsMsg(RsNxsMsg* item, void* data, uint32_t* size);
+    virtual RsNxsMsg* deserialNxsMsg(void *data, uint32_t *size);
+
+    /* RS_PKT_SUBTYPE_NXS_TRANS */
+    virtual uint32_t sizeNxsTrans(RsNxsTransac* item);
+    virtual bool serialiseNxsTrans(RsNxsTransac* item, void* data, uint32_t* size);
+    virtual RsNxsTransac* deserialNxsTrans(void* data, uint32_t *size);
+
+    /* RS_PKT_SUBTYPE_EXTENDED */
+    virtual RsNxsExtended* deserialNxsExtended(void* data, uint32_t *size);
+    virtual uint32_t sizeNxsExtended(RsNxsExtended* item);
+    virtual bool serialiseNxsExtended(RsNxsExtended* item, void* data, uint32_t* size);
+
+private:
+
+    const uint16_t SERVICE_TYPE;
+};
+
+
+#endif // RSNXSITEMS_H

libretroshare/src/serialiser/rsphotov2items.cc

@@ -0,0 +1,45 @@
+/*
+ * rsphotov2items.cc
+ *
+ *  Created on: 22 Jul 2012
+ *      Author: crispy
+ */
+
+#include "rsphotov2items.h"
+
+
+
+
+uint32_t RsGxsPhotoSerialiser::size(RsItem* item) {
+
+}
+
+bool RsGxsPhotoSerialiser::serialise(RsItem* item, void* data, uint32_t* size) {
+}
+
+RsItem* RsGxsPhotoSerialiser::deserialise(void* data, uint32_t* size) {
+}
+
+uint32_t RsGxsPhotoSerialiser::sizeGxsPhotoAlbumItem(RsGxsPhotoAlbumItem* item) {
+
+
+}
+
+bool RsGxsPhotoSerialiser::serialiseGxsPhotoAlbumItem(RsGxsPhotoAlbumItem* item, void* data,
+		uint32_t* size) {
+}
+
+RsGxsPhotoAlbumItem* RsGxsPhotoSerialiser::deserialiseGxsPhotoAlbumItem(void* data,
+		uint32_t* size) {
+}
+
+uint32_t RsGxsPhotoSerialiser::sizeGxsPhotoPhotoItem(RsGxsPhotoPhotoItem* item) {
+}
+
+bool RsGxsPhotoSerialiser::serialiseGxsPhotoPhotoItem(RsGxsPhotoPhotoItem* item, void* data,
+		uint32_t* size) {
+}
+
+RsGxsPhotoPhotoItem* RsGxsPhotoSerialiser::deserialiseGxsPhotoPhotoItem(void* data,
+		uint32_t* size) {
+}

libretroshare/src/serialiser/rsphotov2items.h

@@ -0,0 +1,80 @@
+/*
+ * libretroshare/src/retroshare: rsphoto.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2012-2012 by Christopher Evi-Parker, Robert Fernie
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef RSPHOTOV2ITEMS_H_
+#define RSPHOTOV2ITEMS_H_
+
+#include <map>
+
+#include "serialiser/rsserviceids.h"
+#include "serialiser/rsserial.h"
+#include "serialiser/rstlvtypes.h"
+
+#include "rsgxsitems.h"
+#include "retroshare/rsphotoV2.h"
+
+
+class RsGxsPhotoAlbumItem : public RsGxsGrpItem
+{
+
+public:
+
+	RsGxsPhotoAlbumItem() {}
+	RsPhotoAlbum album;
+};
+
+class RsGxsPhotoPhotoItem : public RsGxsMsgItem
+{
+public:
+
+	RsGxsPhotoPhotoItem() {}
+	RsPhotoPhoto photo;
+};
+
+class RsGxsPhotoSerialiser : public RsSerialType
+{
+	RsGxsPhotoSerialiser()
+	:RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_PHOTO)
+	{ return; }
+	virtual     ~RsGxsPhotoSerialiser() { return; }
+
+	uint32_t    size(RsItem *item);
+	bool        serialise  (RsItem *item, void *data, uint32_t *size);
+	RsItem *    deserialise(void *data, uint32_t *size);
+
+	private:
+
+	uint32_t    sizeGxsPhotoAlbumItem(RsGxsPhotoAlbumItem *item);
+	bool        serialiseGxsPhotoAlbumItem  (RsGxsPhotoAlbumItem *item, void *data, uint32_t *size);
+	RsGxsPhotoAlbumItem *    deserialiseGxsPhotoAlbumItem(void *data, uint32_t *size);
+
+	uint32_t    sizeGxsPhotoPhotoItem(RsGxsPhotoPhotoItem *item);
+	bool        serialiseGxsPhotoPhotoItem  (RsGxsPhotoPhotoItem *item, void *data, uint32_t *size);
+	RsGxsPhotoPhotoItem *    deserialiseGxsPhotoPhotoItem(void *data, uint32_t *size);
+
+
+};
+
+#endif /* RSPHOTOV2ITEMS_H_ */

libretroshare/src/serialiser/rsserviceids.h

@@ -61,7 +61,11 @@ const uint16_t RS_SERVICE_TYPE_CHANNEL_SOCKET = 0xf140;
  /* Status - Service only */
 const uint16_t RS_SERVICE_TYPE_STATUS      = 0xf020;
 
-/* Combined Cache/Service ids */
+/***************** IDS ALLOCATED FOR PLUGINS ******************/
+
+const uint16_t RS_SERVICE_TYPE_PLUGIN_ARADO_ID   = 0x0401;
+const uint16_t RS_SERVICE_TYPE_PLUGIN_QCHESS_ID  = 0x0402;
+
 
 /****************** BELOW ARE ONLY THEORETICAL (CAN BE CHANGED) *****/
 /*
@@ -85,12 +89,17 @@ const uint16_t RS_SERVICE_TYPE_STATUS      = 0xf020;
  /*! for Qblog service (Cache Only) */
 const uint16_t RS_SERVICE_TYPE_QBLOG	   = 0xf010;
 
+/* TEST VOIP - Service only */
+// NOT SURE WHATS HAPPENING WITH THIS ONE? 
+// SHOULD BE DEFINED IN PLUGIN SECTION!
+//const uint16_t RS_SERVICE_TYPE_VOIP        = 0xf011;
+
+ /* Status - Service only */
+//const uint16_t RS_SERVICE_TYPE_STATUS      = 0xf020;
+
  /* Proxy - Service only */
 const uint16_t RS_SERVICE_TYPE_PROXY       = 0xf030;
 
- /* Photo - Cache Only */
-const uint16_t RS_SERVICE_TYPE_PHOTO       = 0xf040;
-
 /* DSDV Testing at the moment - Service Only */
 const uint16_t RS_SERVICE_TYPE_DSDV        = 0xf050;
 
@@ -98,6 +107,19 @@ const uint16_t RS_SERVICE_TYPE_DSDV        = 0xf050;
 const uint16_t RS_SERVICE_TYPE_BWCTRL      = 0xf060;
 
 
+ /* New Cache Services  */
+const uint16_t RS_SERVICE_TYPE_IDENTITY    = 0xf100;
+const uint16_t RS_SERVICE_TYPE_PHOTO       = 0xf101;
+const uint16_t RS_SERVICE_TYPE_WIKI        = 0xf102;
+const uint16_t RS_SERVICE_TYPE_WIRE        = 0xf103;
+const uint16_t RS_SERVICE_TYPE_FORUMSV2    = 0xf104;
+const uint16_t RS_SERVICE_TYPE_POSTED      = 0xf105;
+
+//const uint16_t RS_SERVICE_TYPE_DISTRIB     = 0xf110;
+//const uint16_t RS_SERVICE_TYPE_FORUM       = 0xf120;
+//const uint16_t RS_SERVICE_TYPE_CHANNEL     = 0xf130;
+//const uint16_t RS_SERVICE_TYPE_CHANNEL_SOCKET = 0xf140;
+
 /* Games/External Apps - Service Only */
 const uint16_t RS_SERVICE_TYPE_GAME_LAUNCHER = 0xf200;
 const uint16_t RS_SERVICE_TYPE_PORT          = 0xf201;
@@ -111,8 +133,17 @@ const uint16_t RS_SERVICE_TYPE_GAME_QGO      = 0xf212;
 const uint16_t RS_SERVICE_TYPE_GAME_BIGTWO   = 0xf213;
 const uint16_t RS_SERVICE_TYPE_GAME_POKER    = 0xf214;
 
+/* Rs Network Exchange Service */
+const uint16_t RS_SERVICE_TYPE_NXS = 0xf300;
 
 
+/***************** IDS ALLOCATED FOR PLUGINS ******************/
+
+const uint16_t RS_SERVICE_TYPE_PLUGIN_ARADO_TEST_ID1   = 0xf401;
+const uint16_t RS_SERVICE_TYPE_PLUGIN_QCHESS_TEST_ID1   = 0xf402;
+
+// test
+const uint16_t RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM = 0xf403;
 
 
 

libretroshare/src/serialiser/rstlvtypes.cc

@@ -76,11 +76,22 @@ RsTlvBinaryData::RsTlvBinaryData(uint16_t t)
 	return;
 }
 
+RsTlvBinaryData::RsTlvBinaryData(const RsTlvBinaryData &b)
+    : tlvtype(b.tlvtype), bin_data(NULL), bin_len(0) {
+
+    setBinData(b.bin_data, b.bin_len);
+}
+
 RsTlvBinaryData::~RsTlvBinaryData()
 {
 	TlvClear();
 }
 
+void RsTlvBinaryData::operator =(const RsTlvBinaryData& b){
+
+    setBinData(b.bin_data, b.bin_len);
+    tlvtype = b.tlvtype;
+}
 
 /// used to allocate memory andinitialize binary data member 
 bool     RsTlvBinaryData::setBinData(const void *data, uint32_t size)

libretroshare/src/serialiser/rstlvtypes.h

@@ -65,7 +65,9 @@ class RsTlvBinaryData: public RsTlvItem
 {
 	public:
 	 RsTlvBinaryData(uint16_t t);
-virtual ~RsTlvBinaryData();
+         RsTlvBinaryData(const RsTlvBinaryData& b); // as per rule of three
+         void operator=(const RsTlvBinaryData& b); // as per rule of three
+virtual ~RsTlvBinaryData(); // as per rule of three
 virtual	uint32_t TlvSize();
 virtual	void	 TlvClear(); /*! Initialize fields to empty legal values ( "0", "", etc) */
 virtual	void	 TlvShallowClear(); /*! Don't delete the binary data */

libretroshare/src/services/p3forumsv2.cc

@@ -0,0 +1,805 @@
+/*
+ * libretroshare/src/services p3forumsv2.cc
+ *
+ * ForumsV2 interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "services/p3forumsv2.h"
+
+#include "util/rsrandom.h"
+#include <stdio.h>
+
+/****
+ * #define FORUMV2_DEBUG 1
+ ****/
+
+RsForumsV2 *rsForumsV2 = NULL;
+
+
+
+/********************************************************************************/
+/******************* Startup / Tick    ******************************************/
+/********************************************************************************/
+
+p3ForumsV2::p3ForumsV2(uint16_t type)
+	:p3GxsDataService(type, new ForumDataProxy()), mForumMtx("p3ForumsV2"), mUpdated(true)
+{
+	{
+     		RsStackMutex stack(mForumMtx); /********** STACK LOCKED MTX ******/
+
+		mForumProxy = (ForumDataProxy *) mProxy;
+	}
+
+	generateDummyData();
+
+	return;
+}
+
+
+int	p3ForumsV2::tick()
+{
+	std::cerr << "p3ForumsV2::tick()";
+	std::cerr << std::endl;
+
+	fakeprocessrequests();
+	
+	return 0;
+}
+
+bool p3ForumsV2::updated()
+{
+	RsStackMutex stack(mForumMtx); /********** STACK LOCKED MTX ******/
+
+	if (mUpdated)
+	{
+		mUpdated = false;
+		return true;
+	}
+	return false;
+}
+
+
+
+       /* Data Requests */
+bool p3ForumsV2::requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds)
+{
+	generateToken(token);
+	std::cerr << "p3ForumsV2::requestGroupInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_GROUPS, groupIds);
+
+	return true;
+}
+
+bool p3ForumsV2::requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds)
+{
+	generateToken(token);
+	std::cerr << "p3ForumsV2::requestMsgInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGS, groupIds);
+
+	return true;
+}
+
+bool p3ForumsV2::requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds)
+{
+	generateToken(token);
+	std::cerr << "p3ForumsV2::requestMsgRelatedInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGRELATED, msgIds);
+
+	return true;
+}
+
+        /* Generic Lists */
+bool p3ForumsV2::getGroupList(         const uint32_t &token, std::list<std::string> &groupIds)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_LIST)
+	{
+		std::cerr << "p3ForumsV2::getGroupList() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3ForumsV2::getGroupList() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3ForumsV2::getGroupList() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	bool ans = loadRequestOutList(token, groupIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	return ans;
+}
+
+
+
+
+bool p3ForumsV2::getMsgList(           const uint32_t &token, std::list<std::string> &msgIds)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_LIST)
+	{
+		std::cerr << "p3ForumsV2::getMsgList() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3ForumsV2::getMsgList() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3ForumsV2::getMsgList() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	bool ans = loadRequestOutList(token, msgIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	return ans;
+}
+
+
+        /* Generic Summary */
+bool p3ForumsV2::getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_SUMMARY)
+	{
+		std::cerr << "p3ForumsV2::getGroupSummary() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3ForumsV2::getGroupSummary() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3ForumsV2::getGroupSummary() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	std::list<std::string> groupIds;
+	bool ans = loadRequestOutList(token, groupIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	/* convert to RsGroupMetaData */
+	mProxy->getGroupSummary(groupIds, groupInfo);
+
+	return ans;
+}
+
+bool p3ForumsV2::getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_SUMMARY)
+	{
+		std::cerr << "p3ForumsV2::getMsgSummary() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3ForumsV2::getMsgSummary() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3ForumsV2::getMsgSummary() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	std::list<std::string> msgIds;
+	bool ans = loadRequestOutList(token, msgIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	/* convert to RsMsgMetaData */
+	mProxy->getMsgSummary(msgIds, msgInfo);
+
+	return ans;
+}
+
+
+        /* Specific Service Data */
+bool p3ForumsV2::getGroupData(const uint32_t &token, RsForumV2Group &group)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+	
+
+	if (anstype != RS_TOKREQ_ANSTYPE_DATA)
+	{
+		std::cerr << "p3ForumsV2::getGroupData() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3ForumsV2::getGroupData() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3ForumsV2::getGroupData() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+	
+	std::string id;
+	if (!popRequestOutList(token, id))
+	{
+		/* finished */
+		updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		return false;
+	}
+	
+	/* convert to RsForumAlbum */
+	bool ans = mForumProxy->getForumGroup(id, group);
+	return ans;
+}
+
+
+bool p3ForumsV2::getMsgData(const uint32_t &token, RsForumV2Msg &msg)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+	
+
+	if (anstype != RS_TOKREQ_ANSTYPE_DATA)
+	{
+		std::cerr << "p3ForumsV2::getMsgData() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3ForumsV2::getMsgData() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3ForumsV2::getMsgData() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+	
+	std::string id;
+	if (!popRequestOutList(token, id))
+	{
+		/* finished */
+		updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		return false;
+	}
+	
+	/* convert to RsForumAlbum */
+	bool ans = mForumProxy->getForumMsg(id, msg);
+	return ans;
+}
+
+
+
+        /* Poll */
+uint32_t p3ForumsV2::requestStatus(const uint32_t token)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	return status;
+}
+
+
+        /* Cancel Request */
+bool p3ForumsV2::cancelRequest(const uint32_t &token)
+{
+	return clearRequest(token);
+}
+
+        //////////////////////////////////////////////////////////////////////////////
+
+bool p3ForumsV2::setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask)
+{
+	return mForumProxy->setMessageStatus(msgId, status, statusMask);
+}
+
+bool p3ForumsV2::setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask)
+{
+	return mForumProxy->setGroupStatus(groupId, status, statusMask);
+}
+
+bool p3ForumsV2::setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask)
+{
+	return mForumProxy->setGroupSubscribeFlags(groupId, subscribeFlags, subscribeMask);
+}
+
+bool p3ForumsV2::setMessageServiceString(const std::string &msgId, const std::string &str)
+{
+	return mForumProxy->setMessageServiceString(msgId, str);
+}
+
+bool p3ForumsV2::setGroupServiceString(const std::string &grpId, const std::string &str)
+{
+	return mForumProxy->setGroupServiceString(grpId, str);
+}
+
+
+
+bool p3ForumsV2::groupRestoreKeys(const std::string &groupId)
+{
+	return false;
+}
+
+bool p3ForumsV2::groupShareKeys(const std::string &groupId, std::list<std::string>& peers)
+{
+	return false;
+}
+
+
+
+
+/********************************************************************************************/
+
+	
+std::string p3ForumsV2::genRandomId()
+{
+	std::string randomId;
+	for(int i = 0; i < 20; i++)
+	{
+		randomId += (char) ('a' + (RSRandom::random_u32() % 26));
+	}
+	
+	return randomId;
+}
+	
+bool p3ForumsV2::createGroup(uint32_t &token, RsForumV2Group &group, bool isNew)
+{
+	if (group.mMeta.mGroupId.empty())
+	{
+		/* new photo */
+
+		/* generate a temp id */
+		group.mMeta.mGroupId = genRandomId();
+	}
+	else
+	{
+		std::cerr << "p3ForumsV2::createGroup() Group with existing Id... dropping";
+		std::cerr << std::endl;
+		return false;
+	}
+
+	{	
+		RsStackMutex stack(mForumMtx); /********** STACK LOCKED MTX ******/
+
+		mUpdated = true;
+		mForumProxy->addForumGroup(group);
+	}
+	
+	// Fake a request to return the GroupMetaData.
+	generateToken(token);
+	uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY;
+	RsTokReqOptions opts; // NULL is good.
+	std::list<std::string> groupIds;
+	groupIds.push_back(group.mMeta.mGroupId); // It will just return this one.
+	
+	std::cerr << "p3ForumsV2::createGroup() Generating Request Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_GROUPS, groupIds);
+	
+	return true;
+}
+
+
+
+
+bool p3ForumsV2::createMsg(uint32_t &token, RsForumV2Msg &msg, bool isNew)
+{
+	if (msg.mMeta.mGroupId.empty())
+	{
+		/* new photo */
+		std::cerr << "p3ForumsV2::createForumMsg() Missing MsgID";
+		std::cerr << std::endl;
+		return false;
+	}
+	
+	/* check if its a mod or new msg */
+	if (msg.mMeta.mOrigMsgId.empty())
+	{
+		std::cerr << "p3ForumsV2::createForumMsg() New Msg";
+		std::cerr << std::endl;
+
+		/* new msg, generate a new OrigMsgId */
+		msg.mMeta.mOrigMsgId = genRandomId();
+		msg.mMeta.mMsgId = msg.mMeta.mOrigMsgId;
+	}
+	else
+	{
+		std::cerr << "p3ForumsV2::createForumMsg() Modified Msg";
+		std::cerr << std::endl;
+
+		/* mod msg, keep orig msg id, generate a new MsgId */
+		msg.mMeta.mMsgId = genRandomId();
+	}
+
+	std::cerr << "p3ForumsV2::createForumMsg() GroupId: " << msg.mMeta.mGroupId;
+	std::cerr << std::endl;
+	std::cerr << "p3ForumsV2::createForumMsg() MsgId: " << msg.mMeta.mMsgId;
+	std::cerr << std::endl;
+	std::cerr << "p3ForumsV2::createForumMsg() OrigMsgId: " << msg.mMeta.mOrigMsgId;
+	std::cerr << std::endl;
+
+	{
+		RsStackMutex stack(mForumMtx); /********** STACK LOCKED MTX ******/
+
+		mUpdated = true;
+		mForumProxy->addForumMsg(msg);
+	}
+	
+	// Fake a request to return the MsgMetaData.
+	generateToken(token);
+	uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY;
+	RsTokReqOptions opts; // NULL is good.
+	std::list<std::string> msgIds;
+	msgIds.push_back(msg.mMeta.mMsgId); // It will just return this one.
+	
+	std::cerr << "p3ForumsV2::createMsg() Generating Request Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGRELATED, msgIds);
+
+	return true;
+}
+
+
+
+/********************************************************************************************/
+
+	
+bool ForumDataProxy::getForumGroup(const std::string &id, RsForumV2Group &group)
+{
+	void *groupData = NULL;
+	RsGroupMetaData meta;
+	if (getGroupData(id, groupData) && getGroupSummary(id, meta))
+	{
+		RsForumV2Group *pG = (RsForumV2Group *) groupData;
+		group = *pG;
+
+		// update definitive version of the metadata.
+		group.mMeta = meta;
+
+		std::cerr << "ForumDataProxy::getForumGroup() Id: " << id;
+		std::cerr << " MetaData: " << meta << " DataPointer: " << groupData;
+		std::cerr << std::endl;
+		return true;
+	}
+
+	std::cerr << "ForumDataProxy::getForumGroup() FAILED Id: " << id;
+	std::cerr << std::endl;
+
+	return false;
+}
+
+bool ForumDataProxy::getForumMsg(const std::string &id, RsForumV2Msg &page)
+{
+	void *msgData = NULL;
+	RsMsgMetaData meta;
+	if (getMsgData(id, msgData) && getMsgSummary(id, meta))
+	{
+		RsForumV2Msg *pP = (RsForumV2Msg *) msgData;
+		// Shallow copy of thumbnail.
+		page = *pP;
+	
+		// update definitive version of the metadata.
+		page.mMeta = meta;
+
+		std::cerr << "ForumDataProxy::getForumMsg() Id: " << id;
+		std::cerr << " MetaData: " << meta << " DataPointer: " << msgData;
+		std::cerr << std::endl;
+		return true;
+	}
+
+	std::cerr << "ForumDataProxy::getForumMsg() FAILED Id: " << id;
+	std::cerr << std::endl;
+
+	return false;
+}
+
+bool ForumDataProxy::addForumGroup(const RsForumV2Group &group)
+{
+	// Make duplicate.
+	RsForumV2Group *pG = new RsForumV2Group();
+	*pG = group;
+
+	std::cerr << "ForumDataProxy::addForumGroup()";
+	std::cerr << " MetaData: " << pG->mMeta << " DataPointer: " << pG;
+	std::cerr << std::endl;
+
+	return createGroup(pG);
+}
+
+
+bool ForumDataProxy::addForumMsg(const RsForumV2Msg &msg)
+{
+	// Make duplicate.
+	RsForumV2Msg *pM = new RsForumV2Msg();
+	*pM = msg;
+
+	std::cerr << "ForumDataProxy::addForumMsg()";
+	std::cerr << " MetaData: " << pM->mMeta << " DataPointer: " << pM;
+	std::cerr << std::endl;
+
+	return createMsg(pM);
+}
+
+
+
+        /* These Functions must be overloaded to complete the service */
+bool ForumDataProxy::convertGroupToMetaData(void *groupData, RsGroupMetaData &meta)
+{
+	RsForumV2Group *group = (RsForumV2Group *) groupData;
+	meta = group->mMeta;
+
+	return true;
+}
+
+bool ForumDataProxy::convertMsgToMetaData(void *msgData, RsMsgMetaData &meta)
+{
+	RsForumV2Msg *page = (RsForumV2Msg *) msgData;
+	meta = page->mMeta;
+
+	return true;
+}
+
+
+/********************************************************************************************/
+
+
+
+bool p3ForumsV2::generateDummyData()
+{
+	/* so we want to generate 100's of forums */
+#define MAX_FORUMS 10 //100
+#define MAX_THREADS 10 //1000
+#define MAX_MSGS 100 //10000
+
+	std::list<RsForumV2Group> mGroups;
+	std::list<RsForumV2Group>::iterator git;
+
+	std::list<RsForumV2Msg> mMsgs;
+	std::list<RsForumV2Msg>::iterator mit;
+
+#define DUMMY_NAME_MAX_LEN		10000
+	char name[DUMMY_NAME_MAX_LEN];
+	int i, j;
+	time_t now = time(NULL);
+
+	for(i = 0; i < MAX_FORUMS; i++)
+	{
+		/* generate a new forum */
+		RsForumV2Group forum;
+
+		/* generate a temp id */
+		forum.mMeta.mGroupId = genRandomId();
+
+		snprintf(name, DUMMY_NAME_MAX_LEN, "TestForum_%d", i+1);
+
+		forum.mMeta.mGroupId = genRandomId();
+		forum.mMeta.mGroupName = name;
+
+		forum.mMeta.mPublishTs = now - (RSRandom::random_f32() * 100000);
+		/* key fields to fill in:
+		 * GroupId.
+		 * Name.
+		 * Flags.
+		 * Pop.
+		 */
+
+
+
+		/* use probability to decide which are subscribed / own / popularity.
+		 */
+
+		float rnd = RSRandom::random_f32();
+		if (rnd < 0.1)
+		{
+			forum.mMeta.mSubscribeFlags = RSGXS_GROUP_SUBSCRIBE_ADMIN;
+
+		}
+		else if (rnd < 0.3)
+		{
+			forum.mMeta.mSubscribeFlags = RSGXS_GROUP_SUBSCRIBE_SUBSCRIBED;
+		}
+		else
+		{
+			forum.mMeta.mSubscribeFlags = 0;
+		}
+
+		forum.mMeta.mPop = (int) (RSRandom::random_f32() * 10.0);
+
+		mGroups.push_back(forum);
+
+
+		//std::cerr << "p3ForumsV2::generateDummyData() Generated Forum: " << forum.mMeta;
+		//std::cerr << std::endl;
+	}
+
+
+	for(i = 0; i < MAX_THREADS; i++)
+	{
+		/* generate a base thread */
+
+		/* rotate the Forum Groups Around, then pick one.
+		 */
+
+		int rnd = (int) (RSRandom::random_f32() * 10.0);
+
+		for(j = 0; j < rnd; j++)
+		{
+			RsForumV2Group head = mGroups.front();
+			mGroups.pop_front();
+			mGroups.push_back(head);
+		}
+
+		RsForumV2Group forum = mGroups.front();
+
+		/* now create a new thread */
+
+		RsForumV2Msg msg;
+
+		/* fill in key data 
+		 * GroupId
+		 * MsgId
+		 * OrigMsgId
+		 * ThreadId
+		 * ParentId
+		 * PublishTS (take Forum TS + a bit ).
+		 *
+		 * ChildTS ????
+		 */
+		snprintf(name, DUMMY_NAME_MAX_LEN, "%s => ThreadMsg_%d", forum.mMeta.mGroupName.c_str(), i+1);
+		msg.mMeta.mMsgName = name;
+
+		msg.mMeta.mGroupId = forum.mMeta.mGroupId;
+		msg.mMeta.mMsgId = genRandomId();
+		msg.mMeta.mOrigMsgId = msg.mMeta.mMsgId;
+		msg.mMeta.mThreadId = msg.mMeta.mMsgId;
+		msg.mMeta.mParentId = "";
+
+		msg.mMeta.mPublishTs = forum.mMeta.mPublishTs + (RSRandom::random_f32() * 10000);
+		if (msg.mMeta.mPublishTs > now)
+			msg.mMeta.mPublishTs = now - 1;
+
+		mMsgs.push_back(msg);
+
+		//std::cerr << "p3ForumsV2::generateDummyData() Generated Thread: " << msg.mMeta;
+		//std::cerr << std::endl;
+		
+	}
+
+	for(i = 0; i < MAX_MSGS; i++)
+	{
+		/* generate a base thread */
+
+		/* rotate the Forum Groups Around, then pick one.
+		 */
+
+		int rnd = (int) (RSRandom::random_f32() * 10.0);
+
+		for(j = 0; j < rnd; j++)
+		{
+			RsForumV2Msg head = mMsgs.front();
+			mMsgs.pop_front();
+			mMsgs.push_back(head);
+		}
+
+		RsForumV2Msg parent = mMsgs.front();
+
+		/* now create a new child msg */
+
+		RsForumV2Msg msg;
+
+		/* fill in key data 
+		 * GroupId
+		 * MsgId
+		 * OrigMsgId
+		 * ThreadId
+		 * ParentId
+		 * PublishTS (take Forum TS + a bit ).
+		 *
+		 * ChildTS ????
+		 */
+		snprintf(name, DUMMY_NAME_MAX_LEN, "%s => Msg_%d", parent.mMeta.mMsgName.c_str(), i+1);
+		msg.mMeta.mMsgName = name;
+		msg.mMsg = name;
+
+		msg.mMeta.mGroupId = parent.mMeta.mGroupId;
+		msg.mMeta.mMsgId = genRandomId();
+		msg.mMeta.mOrigMsgId = msg.mMeta.mMsgId;
+		msg.mMeta.mThreadId = parent.mMeta.mThreadId;
+		msg.mMeta.mParentId = parent.mMeta.mOrigMsgId;
+
+		msg.mMeta.mPublishTs = parent.mMeta.mPublishTs + (RSRandom::random_f32() * 10000);
+		if (msg.mMeta.mPublishTs > now)
+			msg.mMeta.mPublishTs = now - 1;
+
+		mMsgs.push_back(msg);
+
+		//std::cerr << "p3ForumsV2::generateDummyData() Generated Child Msg: " << msg.mMeta;
+		//std::cerr << std::endl;
+
+	}
+
+
+	mUpdated = true;
+
+	/* Then - at the end, we push them all into the Proxy */
+	for(git = mGroups.begin(); git != mGroups.end(); git++)
+	{
+		/* pushback */
+		mForumProxy->addForumGroup(*git);
+
+	}
+
+	for(mit = mMsgs.begin(); mit != mMsgs.end(); mit++)
+	{
+		/* pushback */
+		mForumProxy->addForumMsg(*mit);
+	}
+
+	return true;
+}
+

libretroshare/src/services/p3forumsv2.h

@@ -0,0 +1,126 @@
+/*
+ * libretroshare/src/services: p3forumsv2.h
+ *
+ * Wiki interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef P3_FORUMSV2_SERVICE_HEADER
+#define P3_FORUMSV2_SERVICE_HEADER
+
+#include "services/p3gxsservice.h"
+
+#include "retroshare/rsforumsv2.h"
+
+#include <map>
+#include <string>
+
+/* 
+ *
+ */
+
+class ForumDataProxy: public GxsDataProxy
+{
+        public:
+
+        bool getForumGroup(const std::string &id, RsForumV2Group &group);
+        bool getForumMsg(const std::string &id, RsForumV2Msg &msg);
+
+        bool addForumGroup(const RsForumV2Group &group);
+        bool addForumMsg(const RsForumV2Msg &msg);
+
+        /* These Functions must be overloaded to complete the service */
+virtual bool convertGroupToMetaData(void *groupData, RsGroupMetaData &meta);
+virtual bool convertMsgToMetaData(void *msgData, RsMsgMetaData &meta);
+
+};
+
+
+
+
+
+class p3ForumsV2: public p3GxsDataService, public RsForumsV2
+{
+	public:
+
+	p3ForumsV2(uint16_t type);
+
+virtual int	tick();
+
+	public:
+
+
+virtual bool updated();
+
+
+       /* Data Requests */
+virtual bool requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds);
+
+        /* Generic Lists */
+virtual bool getGroupList(         const uint32_t &token, std::list<std::string> &groupIds);
+virtual bool getMsgList(           const uint32_t &token, std::list<std::string> &msgIds);
+
+        /* Generic Summary */
+virtual bool getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo);
+virtual bool getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo);
+
+        /* Actual Data -> specific to Interface */
+        /* Specific Service Data */
+virtual bool getGroupData(const uint32_t &token, RsForumV2Group &group);
+virtual bool getMsgData(const uint32_t &token, RsForumV2Msg &msg);
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token);
+
+        /* Cancel Request */
+virtual bool cancelRequest(const uint32_t &token);
+
+        //////////////////////////////////////////////////////////////////////////////
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str);
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str);
+
+virtual bool groupRestoreKeys(const std::string &groupId);
+virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers);
+
+virtual bool createGroup(uint32_t &token, RsForumV2Group &group, bool isNew);
+virtual bool createMsg(uint32_t &token, RsForumV2Msg &msg, bool isNew);
+
+	private:
+
+std::string genRandomId();
+bool 	generateDummyData();
+
+	ForumDataProxy *mForumProxy;
+
+	RsMutex mForumMtx;
+
+	/***** below here is locked *****/
+
+	bool mUpdated;
+
+};
+
+#endif 

libretroshare/src/services/p3gxsservice.h

@@ -0,0 +1,191 @@
+/*
+ * libretroshare/src/services p3gxsservice.h
+ *
+ * Generic Service Support Class for RetroShare.
+ *
+ * Copyright 2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef P3_GXS_SERVICE_HEADER
+#define P3_GXS_SERVICE_HEADER
+
+#include "services/p3service.h"
+#include "retroshare/rsidentity.h"
+
+/* 
+ * This class provides useful generic support for GXS style services.
+ * I expect much of this will be incorporated into the base GXS.
+ *
+ */
+
+#define GXS_REQUEST_STATUS_FAILED		0
+#define GXS_REQUEST_STATUS_PENDING		1
+#define GXS_REQUEST_STATUS_PARTIAL		2
+#define GXS_REQUEST_STATUS_FINISHED_INCOMPLETE	3
+#define GXS_REQUEST_STATUS_COMPLETE		4
+#define GXS_REQUEST_STATUS_DONE			5 // ONCE ALL DATA RETRIEVED.
+
+#define GXS_REQUEST_TYPE_GROUPS			0x00010000
+#define GXS_REQUEST_TYPE_MSGS			0x00020000
+#define GXS_REQUEST_TYPE_MSGRELATED		0x00040000
+
+class gxsRequest
+{
+	public:
+	uint32_t token;
+	uint32_t reqTime;
+
+	uint32_t ansType; 
+	uint32_t reqType; 
+	RsTokReqOptions Options;
+	
+	uint32_t status;
+
+	std::list<std::string> inList;
+	std::list<std::string> outList;
+	//std::map<std::string, void *> readyData;
+};
+
+
+class p3GxsService: public p3Service
+{
+	protected:
+
+	p3GxsService(uint16_t type);
+
+	public:
+
+//virtual ~p3Service() { p3Service::~p3Service(); return; }
+
+bool	generateToken(uint32_t &token);
+bool	storeRequest(const uint32_t &token, const uint32_t &ansType, const RsTokReqOptions &opts, const uint32_t &type, const std::list<std::string> &ids);
+bool	clearRequest(const uint32_t &token);
+
+bool	updateRequestStatus(const uint32_t &token, const uint32_t &status);
+bool	updateRequestInList(const uint32_t &token, std::list<std::string> ids);
+bool	updateRequestOutList(const uint32_t &token, std::list<std::string> ids);
+//bool	updateRequestData(const uint32_t &token, std::map<std::string, void *> data);
+bool    checkRequestStatus(const uint32_t &token, uint32_t &status, uint32_t &reqtype, uint32_t &anstype, time_t &ts);
+
+	// special ones for testing (not in final design)
+bool    tokenList(std::list<uint32_t> &tokens);
+bool    popRequestInList(const uint32_t &token, std::string &id);
+bool    popRequestOutList(const uint32_t &token, std::string &id);
+bool    loadRequestOutList(const uint32_t &token, std::list<std::string> &ids);
+
+virtual bool    fakeprocessrequests();
+
+	protected:
+
+	RsMutex mReqMtx;
+
+	uint32_t mNextToken;
+
+	std::map<uint32_t, gxsRequest> mRequests;
+
+};
+
+
+class GxsDataProxy
+{
+	public:
+
+	GxsDataProxy();
+
+virtual bool getGroupList(     uint32_t &token, const RsTokReqOptions &opts, const std::list<std::string> &groupIds, std::list<std::string> &outGroupIds);
+virtual bool getMsgList(       uint32_t &token, const RsTokReqOptions &opts, const std::list<std::string> &groupIds, std::list<std::string> &outMsgIds);
+virtual bool getMsgRelatedList(uint32_t &token, const RsTokReqOptions &opts, const std::list<std::string> &msgIds, std::list<std::string> &outMsgIds);
+
+
+	/* This functions return a token - which can be used to retrieve the RsGroupMetaData, later
+	 * This is required, as signatures and keys might have to be generated in the background
+	 * Though at the moment: for this test system it won't change anything? FIXME.
+	 */
+virtual bool createGroup(void *groupData);
+virtual bool createMsg(void *msgData);
+
+	/* These Functions must be overloaded to complete the service */
+virtual bool convertGroupToMetaData(void *groupData, RsGroupMetaData &meta);
+virtual bool convertMsgToMetaData(void *groupData, RsMsgMetaData &meta);
+
+	/* extract Data */
+	bool getGroupSummary(const std::list<std::string> &groupIds, std::list<RsGroupMetaData> &groupSummary);
+	bool getMsgSummary(const std::list<std::string> &msgIds, std::list<RsMsgMetaData> &msgSummary);
+
+	bool getGroupSummary(const std::string &groupId, RsGroupMetaData &groupSummary);
+	bool getMsgSummary(const std::string &msgId, RsMsgMetaData &msgSummary);
+
+	//bool getGroupData(const std::list<std::string> &groupIds, std::list<void *> &groupData);
+	//bool getMsgData(const std::list<std::string> &msgIds, std::list<void *> &msgData);
+	bool getGroupData(const std::string &groupId, void * &groupData);
+	bool getMsgData(const std::string &msgId, void * &msgData);
+
+	bool isUniqueGroup(const std::string &groupId);
+	bool isUniqueMsg(const std::string &msgId);
+
+
+	/* Handle Status & Subscribe Modes */
+// This is removed as redundant - use getGroupList - with OptFlags to find these.
+//virtual bool requestGroupsChanged(uint32_t &token); //std::list<std::string> &groupIds);
+
+        // Get Message Status - is retrived via MessageSummary.
+	// These operations could have a token, but for the moment we are going to assume
+	// they are async and always succeed - (or fail silently).
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
+
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str);
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str);
+
+	protected:
+
+	bool filterGroupList(const RsTokReqOptions &opts, std::list<std::string> &groupIds);
+	bool filterMsgList(const RsTokReqOptions &opts, std::list<std::string> &msgIds);
+
+
+	RsMutex mDataMtx;
+	
+	std::map<std::string, void *> mGroupData;
+	std::map<std::string, void *> mMsgData;
+
+	std::map<std::string, RsGroupMetaData> mGroupMetaData;
+	std::map<std::string, RsMsgMetaData>   mMsgMetaData;
+
+};
+
+
+class p3GxsDataService: public p3GxsService
+{
+	public:
+
+	p3GxsDataService(uint16_t type, GxsDataProxy *proxy);
+virtual bool    fakeprocessrequests();
+
+
+	protected:
+
+	GxsDataProxy *mProxy;
+};
+
+
+
+#endif // P3_GXS_SERVICE_HEADER
+

libretroshare/src/services/p3idservice.h

@@ -0,0 +1,133 @@
+/*
+ * libretroshare/src/services: p3idservice.h
+ *
+ * Identity interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef P3_IDENTITY_SERVICE_HEADER
+#define P3_IDENTITY_SERVICE_HEADER
+
+#include "services/p3service.h"
+#include "services/p3gxsservice.h"
+
+#include "retroshare/rsidentity.h"
+
+#include <map>
+#include <string>
+
+/* 
+ * Identity Service
+ *
+ */
+
+class IdDataProxy: public GxsDataProxy
+{
+        public:
+
+        bool getGroup(const std::string &id, RsIdGroup &group);
+        bool getMsg(const std::string &id, RsIdMsg &msg);
+
+        bool addGroup(const RsIdGroup &group);
+        bool addMsg(const RsIdMsg &msg);
+
+        /* These Functions must be overloaded to complete the service */
+virtual bool convertGroupToMetaData(void *groupData, RsGroupMetaData &meta);
+virtual bool convertMsgToMetaData(void *msgData, RsMsgMetaData &meta);
+};
+
+
+class p3IdService: public p3GxsDataService, public RsIdentity
+{
+	public:
+
+	p3IdService(uint16_t type);
+
+virtual int	tick();
+
+	public:
+
+
+        /* changed? */
+virtual bool updated();
+
+	/* From RsTokenService */
+        /* Data Requests */
+virtual bool requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds);
+
+        /* Generic Lists */
+virtual bool getGroupList(         const uint32_t &token, std::list<std::string> &groupIds);
+virtual bool getMsgList(           const uint32_t &token, std::list<std::string> &msgIds);
+
+        /* Generic Summary */
+virtual bool getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo);
+virtual bool getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo);
+
+        /* Actual Data -> specific to Interface */
+virtual bool 	getGroupData(const uint32_t &token, RsIdGroup &group);
+virtual bool 	getMsgData(const uint32_t &token, RsIdMsg &msg);
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token);
+
+        /* Cancel Request */
+virtual bool cancelRequest(const uint32_t &token);
+
+        //////////////////////////////////////////////////////////////////////////////
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str);
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str);
+
+virtual bool groupRestoreKeys(const std::string &groupId);
+virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers);
+
+virtual bool 	createGroup(uint32_t &token, RsIdGroup &group, bool isNew);
+virtual bool 	createMsg(uint32_t &token, RsIdMsg &msg, bool isNew);
+
+
+	private:
+
+virtual void generateDummyData();
+
+std::string genRandomId();
+
+        IdDataProxy *mIdProxy;
+
+	RsMutex mIdMtx;
+
+	/***** below here is locked *****/
+
+	bool mUpdated;
+
+#if 0
+	std::map<std::string, RsIdData> mIds;
+	std::map<std::string, std::map<std::string, RsIdOpinion> >  mOpinions;
+
+	std::map<std::string, RsIdReputation> mReputations; // this is created locally.
+#endif
+
+};
+
+#endif 

libretroshare/src/services/p3photoservice.h

@@ -3,7 +3,7 @@
  *
  * 3P/PQI network interface for RetroShare.
  *
- * Copyright 2008-2008 by Robert Fernie.
+ * Copyright 2012-2012 by Robert Fernie.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Library General Public
@@ -26,117 +26,112 @@
 #ifndef P3_PHOTO_SERVICE_HEADER
 #define P3_PHOTO_SERVICE_HEADER
 
-#include "dbase/cachestrapper.h"
+#include "services/p3gxsservice.h"
-#include "pqi/pqiservice.h"
-#include "pqi/pqistreamer.h"
-
-#include "serialiser/rsserial.h"
-#include "serialiser/rsphotoitems.h"
-
 #include "retroshare/rsphoto.h"
 
+#include <map>
+#include <string>
+
 /* 
  * Photo Service
+ *
+ * This is an example service for the new cache system.
+ * For the moment, it will only hold data passed to it from the GUI.
+ * and spew that back when asked....
+ *
+ * We are doing it like this - so we can check the required interface functionality.
+ *
+ * Expect it won't take long before it'll be properly linked into the backend!
+ *
+ * This will be transformed into a Plugin Service, once the basics have been worked out.
+ *
  */
 
-class PhotoSet
+
+class PhotoDataProxy: public GxsDataProxy
 {
 	public:
 
-	PhotoSet();
+	bool addAlbum(const RsPhotoAlbum &album);
-        std::string pid;
+	bool addPhoto(const RsPhotoPhoto &photo);
+
+	bool getAlbum(const std::string &id, RsPhotoAlbum &album);
+	bool getPhoto(const std::string &id, RsPhotoPhoto &photo);
+
+        /* These Functions must be overloaded to complete the service */
+virtual bool convertGroupToMetaData(void *groupData, RsGroupMetaData &meta);
+virtual bool convertMsgToMetaData(void *groupData, RsMsgMetaData &meta);
 
-        std::map<std::string, RsPhotoItem *> photos;
-        std::map<std::string, RsPhotoShowItem *> shows;
 };
 
 
 
-class p3PhotoService: public CacheSource, public CacheStore
+class p3PhotoService: public p3GxsDataService, public RsPhoto
 {
 	public:
 
-	p3PhotoService(uint16_t type, CacheStrapper *cs, CacheTransfer *cft,
+	p3PhotoService(uint16_t type);
-		std::string sourcedir, std::string storedir);
 
-void	tick();
+virtual int	tick();
-
-/******************************* CACHE SOURCE / STORE Interface *********************/
-
-	/* overloaded functions from Cache Source */
-virtual bool    loadLocalCache(const CacheData &data);
-
-	/* overloaded functions from Cache Store */
-virtual int    loadCache(const CacheData &data);
-
-/******************************* CACHE SOURCE / STORE Interface *********************/
 
 	public:
 
+// NEW INTERFACE.
 /************* Extern Interface *******/
 
-	/* things changed */
+        /* changed? */
-bool updated();
+virtual bool updated();
 
-        /* access data */
+       /* Data Requests */
-bool getPhotoList(std::string id, std::list<std::string> &hashs);
+virtual bool requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
-bool getShowList(std::string id, std::list<std::string> &showIds);
+virtual bool requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
-bool getShowDetails(std::string id, std::string showId, RsPhotoShowDetails &detail);
+virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds);
-bool getPhotoDetails(std::string id, std::string photoId, RsPhotoDetails &detail);
 
-	        /* add / delete */
+        /* Generic Lists */
-std::string createShow(std::string name);
+virtual bool getGroupList(         const uint32_t &token, std::list<std::string> &groupIds);
-bool deleteShow(std::string showId);
+virtual bool getMsgList(           const uint32_t &token, std::list<std::string> &msgIds);
-bool addPhotoToShow(std::string showId, std::string photoId, int16_t index);
-bool movePhotoInShow(std::string showId, std::string photoId, int16_t index);
-bool removePhotoFromShow(std::string showId, std::string photoId);
 
-std::string addPhoto(std::string path); /* add from file */
+        /* Generic Summary */
-bool addPhoto(std::string srcId, std::string photoId); /* add from peers photos */
+virtual bool getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo);
-bool deletePhoto(std::string photoId);
+virtual bool getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo);
 
-	        /* modify properties (TODO) */
+        /* Actual Data -> specific to Interface */
-bool modifyShow(std::string showId, std::wstring name, std::wstring comment);
+        /* Specific Service Data */
-bool modifyPhoto(std::string photoId, std::wstring name, std::wstring comment);
+virtual bool getAlbum(const uint32_t &token, RsPhotoAlbum &album);
-bool modifyShowComment(std::string showId, std::string photoId, std::wstring comment);
+virtual bool getPhoto(const uint32_t &token, RsPhotoPhoto &photo);
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token);
+
+        /* Cancel Request */
+virtual bool cancelRequest(const uint32_t &token);
+
+        //////////////////////////////////////////////////////////////////////////////
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str);
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str);
+
+virtual bool groupRestoreKeys(const std::string &groupId);
+virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers);
+
+
+/* details are updated in album - to choose Album ID, and storage path */
+virtual bool submitAlbumDetails(uint32_t &token, RsPhotoAlbum &album, bool isNew);
+virtual bool submitPhoto(uint32_t &token, RsPhotoPhoto &photo, bool isNew);
 
 
 
-	private:	
+	private:
-
-	/* cache processing */
-
-void loadPhotoIndex(std::string filename, std::string hash, std::string src);
-void availablePhoto(std::string filename, std::string hash, std::string src);
-
-bool loadPhotoItem(RsPhotoItem *item);
-bool loadPhotoShowItem(RsPhotoShowItem *item);
-void publishPhotos();
-
-
-	/* locate info */
-
-PhotoSet 	&locked_getPhotoSet(std::string id);
-RsPhotoItem 	*locked_getPhoto(std::string id, std::string photoId);
-RsPhotoShowItem *locked_getShow(std::string id, std::string showId);
-
-
-	/* test functions */
-void		createDummyData();
 
+std::string genRandomId();
 
+	PhotoDataProxy *mPhotoProxy;
 
 	RsMutex mPhotoMtx;
-
-	/***** below here is locked *****/
-
-	bool mRepublish;
-	std::string mOwnId;
-
 	bool mUpdated;
 
-	std::map<std::string, PhotoSet> mPhotos;
-
 
 };
 

libretroshare/src/services/p3photoserviceV2.cc

@@ -0,0 +1,143 @@
+#include "p3photoserviceV2.h"
+#include "serialiser/rsphotov2items.h"
+
+p3PhotoServiceV2::p3PhotoServiceV2(RsGeneralDataService* gds, RsNetworkExchangeService* nes)
+	: RsGenExchange(gds, nes, new RsGxsPhotoSerialiser(), RS_SERVICE_TYPE_PHOTO)
+{
+
+}
+
+bool p3PhotoServiceV2::updated()
+{
+	return false;
+}
+
+
+void p3PhotoServiceV2::groupsChanged(std::list<std::string>& grpIds) {
+}
+
+
+void p3PhotoServiceV2::msgsChanged(
+		std::map<std::string, std::vector<std::string> >& msgs)
+{
+
+}
+
+
+RsTokenService* p3PhotoServiceV2::getTokenService() {
+
+	return RsGenExchange::getTokenService();
+}
+
+
+bool p3PhotoServiceV2::getGroupList(const uint32_t& token,
+		std::list<std::string>& groupIds)
+{
+	return RsGenExchange::getGroupList(token, groupIds);
+}
+
+
+bool p3PhotoServiceV2::getMsgList(const uint32_t& token,
+		GxsMsgIdResult& msgIds)
+{
+
+	return RsGenExchange::getMsgList(token, msgIds);
+}
+
+
+bool p3PhotoServiceV2::getGroupSummary(const uint32_t& token,
+		std::list<RsGroupMetaData>& groupInfo)
+{
+	return RsGenExchange::getGroupMeta(token, groupInfo);
+}
+
+
+bool p3PhotoServiceV2::getMsgSummary(const uint32_t& token,
+		MsgMetaResult& msgInfo)
+{
+	return RsGenExchange::getMsgMeta(token, msgInfo);
+}
+
+
+bool p3PhotoServiceV2::getAlbum(const uint32_t& token, std::vector<RsPhotoAlbum>& albums)
+{
+	std::vector<RsGxsGrpItem*> grpData;
+	bool ok = RsGenExchange::getGroupData(token, grpData);
+
+	if(ok)
+	{
+		std::vector<RsGxsGrpItem*>::iterator vit = grpData.begin();
+
+		for(; vit != grpData.end(); vit++)
+		{
+			RsGxsGrpItem* item = *vit;
+			RsPhotoAlbum album = *item;
+			albums.push_back(album);
+		}
+	}
+
+	return ok;
+}
+
+
+bool p3PhotoServiceV2::getPhoto(const uint32_t& token, PhotoResult& photo)
+{
+	GxsMsgDataMap msgData;
+	bool ok = RsGenExchange::getMsgData(token, msgData);
+
+	if(ok)
+	{
+		GxsMsgDataMap::iterator mit = msgData.begin();
+
+		for(; mit != msgData.end();  mit++)
+		{
+			RsGxsGroupId grpId = mit->first;
+			std::vector<RsGxsMsgItem*>& msgItems = mit->second;
+			std::vector<RsGxsMsgItem*>::iterator vit = msgItems.begin();
+
+			for(; vit != msgItems.end(); vit++)
+			{
+				RsGxsPhotoPhotoItem* item = dynamic_cast<RsGxsPhotoPhotoItem*>(*vit);
+
+				if(item)
+				{
+					RsPhotoPhoto photo = *item;
+					photo[grpId] = photo;
+					delete item;
+				}else
+				{
+					delete *vit;
+				}
+			}
+		}
+	}
+
+	return ok;
+}
+
+
+bool p3PhotoServiceV2::submitAlbumDetails(RsPhotoAlbum& album)
+{
+	return false;
+}
+
+void p3PhotoServiceV2::operator =(RsPhoto& lPhotos,
+		const RsGxsPhotoPhotoItem& rPhoto)
+{
+	lPhotos = rPhoto.photo;
+}
+
+
+
+void p3PhotoServiceV2::operator =(RsPhotoAlbum& lAlbum,
+		const RsGxsPhotoAlbumItem& rAlbum)
+{
+	lAlbum = rAlbum.album;
+}
+
+bool p3PhotoServiceV2::submitPhoto(RsPhotoPhoto& photo)
+{
+	return false;
+}
+
+

libretroshare/src/services/p3photoserviceV2.h

@@ -0,0 +1,87 @@
+#ifndef P3PHOTOSERVICEV2_H
+#define P3PHOTOSERVICEV2_H
+
+/*
+ * libretroshare/src/retroshare: rsphoto.h
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2008-2012 by Robert Fernie, Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "gxs/rsgenexchange.h"
+#include "retroshare/rsphotoV2.h"
+
+
+class p3PhotoServiceV2 : public RsPhotoV2, public RsGenExchange
+{
+public:
+
+    p3PhotoServiceV2(RsGeneralDataService* gds, RsNetworkExchangeService* nes);
+
+public:
+
+    /*!
+     * @return
+     */
+    bool updated();
+
+public:
+
+    /** Requests **/
+
+    void groupsChanged(std::list<std::string>& grpIds);
+
+
+    void msgsChanged(std::map<std::string,
+                             std::vector<std::string> >& msgs);
+
+    RsTokenService* getTokenService();
+
+    bool getGroupList(const uint32_t &token,
+                              std::list<std::string> &groupIds);
+    bool getMsgList(const uint32_t &token,
+                            GxsMsgIdResult& msgIds);
+
+    /* Generic Summary */
+    bool getGroupSummary(const uint32_t &token,
+                                 std::list<RsGroupMetaData> &groupInfo);
+
+    bool getMsgSummary(const uint32_t &token,
+                               MsgMetaResult &msgInfo);
+
+    /* Specific Service Data */
+    bool getAlbum(const uint32_t &token, std::vector<RsPhotoAlbum> &albums);
+    bool getPhoto(const uint32_t &token, PhotoResult &photos);
+
+private:
+
+    void operator=(RsPhoto& lPhotos, const RsGxsPhotoPhotoItem& rPhoto);
+    void operator=(RsPhotoAlbum& lAlbum, const RsGxsPhotoAlbumItem& rAlbum);
+
+public:
+
+    /** Modifications **/
+
+    bool submitAlbumDetails(RsPhotoAlbum &album);
+    bool submitPhoto(RsPhotoPhoto &photo);
+};
+
+#endif // P3PHOTOSERVICEV2_H

libretroshare/src/services/p3posted.h

@@ -0,0 +1,190 @@
+/*
+ * libretroshare/src/services: p3posted.h
+ *
+ * 3P/PQI network interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef P3_POSTED_SERVICE_HEADER
+#define P3_POSTED_SERVICE_HEADER
+
+#include "services/p3gxsservice.h"
+#include "retroshare/rsposted.h"
+
+#include <map>
+#include <string>
+
+/* 
+ * Posted Service
+ *
+ */
+
+
+class PostedDataProxy: public GxsDataProxy
+{
+	public:
+
+	bool addGroup(const RsPostedGroup &group);
+	bool addPost(const RsPostedPost &post);
+	bool addVote(const RsPostedVote &vote);
+	bool addComment(const RsPostedComment &comment);
+
+	bool getGroup(const std::string &id, RsPostedGroup &group);
+	bool getPost(const std::string &id, RsPostedPost &post);
+	bool getVote(const std::string &id, RsPostedVote &vote);
+	bool getComment(const std::string &id, RsPostedComment &comment);
+
+        /* These Functions must be overloaded to complete the service */
+virtual bool convertGroupToMetaData(void *groupData, RsGroupMetaData &meta);
+virtual bool convertMsgToMetaData(void *groupData, RsMsgMetaData &meta);
+
+};
+
+
+
+class p3PostedService: public p3GxsDataService, public RsPosted
+{
+	public:
+
+	p3PostedService(uint16_t type);
+
+virtual int	tick();
+
+	public:
+
+// NEW INTERFACE.
+/************* Extern Interface *******/
+
+virtual bool updated();
+
+       /* Data Requests */
+virtual bool requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds);
+
+        /* Generic Lists */
+virtual bool getGroupList(         const uint32_t &token, std::list<std::string> &groupIds);
+virtual bool getMsgList(           const uint32_t &token, std::list<std::string> &msgIds);
+
+        /* Generic Summary */
+virtual bool getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo);
+virtual bool getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo);
+
+        /* Actual Data -> specific to Interface */
+        /* Specific Service Data */
+virtual bool getGroup(const uint32_t &token, RsPostedGroup &group);
+virtual bool getPost(const uint32_t &token, RsPostedPost &post);
+virtual bool getComment(const uint32_t &token, RsPostedComment &comment);
+
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token);
+
+        /* Cancel Request */
+virtual bool cancelRequest(const uint32_t &token);
+
+        //////////////////////////////////////////////////////////////////////////////
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str);
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str);
+
+virtual bool groupRestoreKeys(const std::string &groupId);
+virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers);
+
+virtual bool submitGroup(uint32_t &token, RsPostedGroup &group, bool isNew);
+virtual bool submitPost(uint32_t &token, RsPostedPost &post, bool isNew);
+virtual bool submitVote(uint32_t &token, RsPostedVote &vote, bool isNew);
+virtual bool submitComment(uint32_t &token, RsPostedComment &comment, bool isNew);
+
+	// Extended Interface for Collated Data View.
+virtual bool setViewMode(uint32_t mode);
+virtual bool setViewPeriod(uint32_t period);
+virtual bool setViewRange(uint32_t first, uint32_t count);
+
+virtual bool requestRanking(uint32_t &token, std::string groupId);
+virtual bool getRankedPost(const uint32_t &token, RsPostedPost &post);
+
+
+	// These are exposed for GUI usage.
+virtual bool encodePostedCache(std::string &str, uint32_t votes, uint32_t comments);
+virtual bool extractPostedCache(const std::string &str, uint32_t &votes, uint32_t &comments);
+virtual float calcPostScore(const RsMsgMetaData &meta);
+
+	private:
+
+	// 
+bool 	checkRankingRequest();
+bool 	processPosts();
+
+	// background processing of Votes.
+	// NB: These should probably be handled by a background thread.
+	// At the moment they are run from the tick() thread.
+
+bool 	background_checkTokenRequest();
+bool 	background_requestGroups();
+bool 	background_requestNewMessages();
+bool 	background_processNewMessages();
+
+bool 	background_updateVoteCounts();
+bool 	background_cleanup();
+
+
+
+std::string genRandomId();
+bool 	generateDummyData();
+bool 	addExtraDummyData();
+
+	PostedDataProxy *mPostedProxy;
+
+	RsMutex mPostedMtx;
+	bool mUpdated;
+
+	// Ranking view mode, stored here.
+	uint32_t mViewMode;
+	uint32_t mViewPeriod;
+	uint32_t mViewStart;
+	uint32_t mViewCount;
+
+	// Processing Ranking stuff.
+	bool mProcessingRanking;
+	uint32_t mRankingState;
+	uint32_t mRankingExternalToken;
+	uint32_t mRankingInternalToken;
+
+	// background processing - Mutex protected.
+	time_t   mLastBgCheck;
+	bool 	 mBgProcessing;
+	uint32_t mBgPhase;
+	uint32_t mBgToken;
+
+	std::map<std::string, uint32_t> mBgVoteMap;	// ParentId -> Vote Count.
+	std::map<std::string, uint32_t> mBgCommentMap;  // ThreadId -> Comment Count.
+
+	// extra dummy data.
+        std::list<RsPostedVote>    mDummyLaterVotes;
+        std::list<RsPostedComment> mDummyLaterComments;
+
+
+};
+
+#endif 

libretroshare/src/services/p3wikiservice.cc

@@ -0,0 +1,595 @@
+/*
+ * libretroshare/src/services p3wikiservice.cc
+ *
+ * Wiki interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "services/p3wikiservice.h"
+
+#include "util/rsrandom.h"
+
+/****
+ * #define WIKI_DEBUG 1
+ ****/
+
+RsWiki *rsWiki = NULL;
+
+
+/********************************************************************************/
+/******************* Startup / Tick    ******************************************/
+/********************************************************************************/
+
+p3WikiService::p3WikiService(uint16_t type)
+	:p3GxsDataService(type, new WikiDataProxy()), mWikiMtx("p3WikiService"), mUpdated(true)
+{
+     	RsStackMutex stack(mWikiMtx); /********** STACK LOCKED MTX ******/
+
+	mWikiProxy = (WikiDataProxy *) mProxy;
+	return;
+}
+
+
+int	p3WikiService::tick()
+{
+	std::cerr << "p3WikiService::tick()";
+	std::cerr << std::endl;
+
+	fakeprocessrequests();
+	
+	return 0;
+}
+
+bool p3WikiService::updated()
+{
+	RsStackMutex stack(mWikiMtx); /********** STACK LOCKED MTX ******/
+
+	if (mUpdated)
+	{
+		mUpdated = false;
+		return true;
+	}
+	return false;
+}
+
+
+
+       /* Data Requests */
+bool p3WikiService::requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds)
+{
+	generateToken(token);
+	std::cerr << "p3WikiService::requestGroupInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_GROUPS, groupIds);
+
+	return true;
+}
+
+bool p3WikiService::requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds)
+{
+	generateToken(token);
+	std::cerr << "p3WikiService::requestMsgInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGS, groupIds);
+
+	return true;
+}
+
+bool p3WikiService::requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds)
+{
+	generateToken(token);
+	std::cerr << "p3WikiService::requestMsgRelatedInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGRELATED, msgIds);
+
+	return true;
+}
+
+        /* Generic Lists */
+bool p3WikiService::getGroupList(         const uint32_t &token, std::list<std::string> &groupIds)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_LIST)
+	{
+		std::cerr << "p3WikiService::getGroupList() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3WikiService::getGroupList() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3WikiService::getGroupList() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	bool ans = loadRequestOutList(token, groupIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	return ans;
+}
+
+
+
+
+bool p3WikiService::getMsgList(           const uint32_t &token, std::list<std::string> &msgIds)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_LIST)
+	{
+		std::cerr << "p3WikiService::getMsgList() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3WikiService::getMsgList() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3WikiService::getMsgList() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	bool ans = loadRequestOutList(token, msgIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	return ans;
+}
+
+
+        /* Generic Summary */
+bool p3WikiService::getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_SUMMARY)
+	{
+		std::cerr << "p3WikiService::getGroupSummary() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3WikiService::getGroupSummary() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3WikiService::getGroupSummary() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	std::list<std::string> groupIds;
+	bool ans = loadRequestOutList(token, groupIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	/* convert to RsGroupMetaData */
+	mProxy->getGroupSummary(groupIds, groupInfo);
+
+	return ans;
+}
+
+bool p3WikiService::getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_SUMMARY)
+	{
+		std::cerr << "p3WikiService::getMsgSummary() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3WikiService::getMsgSummary() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3WikiService::getMsgSummary() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	std::list<std::string> msgIds;
+	bool ans = loadRequestOutList(token, msgIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	/* convert to RsMsgMetaData */
+	mProxy->getMsgSummary(msgIds, msgInfo);
+
+	return ans;
+}
+
+
+        /* Specific Service Data */
+bool p3WikiService::getGroupData(const uint32_t &token, RsWikiGroup &group)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+	
+
+	if (anstype != RS_TOKREQ_ANSTYPE_DATA)
+	{
+		std::cerr << "p3WikiService::getGroupData() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3WikiService::getGroupData() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3WikiService::getGroupData() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+	
+	std::string id;
+	if (!popRequestOutList(token, id))
+	{
+		/* finished */
+		updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		return false;
+	}
+	
+	/* convert to RsWikiAlbum */
+	bool ans = mWikiProxy->getGroup(id, group);
+	return ans;
+}
+
+
+bool p3WikiService::getMsgData(const uint32_t &token, RsWikiPage &page)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+	
+
+	if (anstype != RS_TOKREQ_ANSTYPE_DATA)
+	{
+		std::cerr << "p3WikiService::getMsgData() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3WikiService::getMsgData() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3WikiService::getMsgData() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+	
+	std::string id;
+	if (!popRequestOutList(token, id))
+	{
+		/* finished */
+		updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		return false;
+	}
+	
+	/* convert to RsWikiAlbum */
+	bool ans = mWikiProxy->getPage(id, page);
+	return ans;
+}
+
+
+
+        /* Poll */
+uint32_t p3WikiService::requestStatus(const uint32_t token)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	return status;
+}
+
+
+        /* Cancel Request */
+bool p3WikiService::cancelRequest(const uint32_t &token)
+{
+	return clearRequest(token);
+}
+
+
+        //////////////////////////////////////////////////////////////////////////////
+bool p3WikiService::setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask)
+{
+	return mWikiProxy->setMessageStatus(msgId, status, statusMask);
+}
+
+bool p3WikiService::setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask)
+{
+	return mWikiProxy->setGroupStatus(groupId, status, statusMask);
+}
+
+bool p3WikiService::setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask)
+{
+	return mWikiProxy->setGroupSubscribeFlags(groupId, subscribeFlags, subscribeMask);
+}
+
+bool p3WikiService::setMessageServiceString(const std::string &msgId, const std::string &str)
+{
+	return mWikiProxy->setMessageServiceString(msgId, str);
+}
+
+bool p3WikiService::setGroupServiceString(const std::string &grpId, const std::string &str)
+{
+	return mWikiProxy->setGroupServiceString(grpId, str);
+}
+
+
+bool p3WikiService::groupRestoreKeys(const std::string &groupId)
+{
+	return false;
+}
+
+bool p3WikiService::groupShareKeys(const std::string &groupId, std::list<std::string>& peers)
+{
+	return false;
+}
+
+
+/********************************************************************************************/
+
+	
+std::string p3WikiService::genRandomId()
+{
+	std::string randomId;
+	for(int i = 0; i < 20; i++)
+	{
+		randomId += (char) ('a' + (RSRandom::random_u32() % 26));
+	}
+	
+	return randomId;
+}
+	
+bool p3WikiService::createGroup(uint32_t &token, RsWikiGroup &group, bool isNew)
+{
+	if (group.mMeta.mGroupId.empty())
+	{
+		/* new photo */
+
+		/* generate a temp id */
+		group.mMeta.mGroupId = genRandomId();
+	}
+	else
+	{
+		std::cerr << "p3WikiService::createGroup() Group with existing Id... dropping";
+		std::cerr << std::endl;
+		return false;
+	}
+
+	{	
+		RsStackMutex stack(mWikiMtx); /********** STACK LOCKED MTX ******/
+
+		mUpdated = true;
+		mWikiProxy->addGroup(group);
+	}
+	
+	// Fake a request to return the GroupMetaData.
+	generateToken(token);
+	uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY;
+	RsTokReqOptions opts; // NULL is good.
+	std::list<std::string> groupIds;
+	groupIds.push_back(group.mMeta.mGroupId); // It will just return this one.
+	
+	std::cerr << "p3WikiService::createGroup() Generating Request Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_GROUPS, groupIds);
+
+	return true;
+}
+
+
+
+bool p3WikiService::createPage(uint32_t &token, RsWikiPage &page, bool isNew)
+{
+	if (page.mMeta.mGroupId.empty())
+	{
+		/* new photo */
+		std::cerr << "p3WikiService::createPage() Missing PageID";
+		std::cerr << std::endl;
+		return false;
+	}
+	
+	/* check if its a mod or new page */
+	if (page.mMeta.mOrigMsgId.empty())
+	{
+		std::cerr << "p3WikiService::createPage() New Page";
+		std::cerr << std::endl;
+
+		/* new page, generate a new OrigPageId */
+		page.mMeta.mOrigMsgId = genRandomId();
+		page.mMeta.mMsgId = page.mMeta.mOrigMsgId;
+	}
+	else
+	{
+		std::cerr << "p3WikiService::createPage() Modified Page";
+		std::cerr << std::endl;
+
+		/* mod page, keep orig page id, generate a new PageId */
+		page.mMeta.mMsgId = genRandomId();
+	}
+
+	std::cerr << "p3WikiService::createPage() GroupId: " << page.mMeta.mGroupId;
+	std::cerr << std::endl;
+	std::cerr << "p3WikiService::createPage() PageId: " << page.mMeta.mMsgId;
+	std::cerr << std::endl;
+	std::cerr << "p3WikiService::createPage() OrigPageId: " << page.mMeta.mOrigMsgId;
+	std::cerr << std::endl;
+
+	{
+		RsStackMutex stack(mWikiMtx); /********** STACK LOCKED MTX ******/
+
+		mUpdated = true;
+		mWikiProxy->addPage(page);
+	}
+
+	// Fake a request to return the MsgMetaData.
+	generateToken(token);
+	uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY;
+	RsTokReqOptions opts; // NULL is good.
+	std::list<std::string> msgIds;
+	msgIds.push_back(page.mMeta.mMsgId); // It will just return this one.
+	
+	std::cerr << "p3WikiService::createPage() Generating Request Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGRELATED, msgIds);
+	
+	return true;
+}
+
+
+
+/********************************************************************************************/
+
+
+	
+bool WikiDataProxy::getGroup(const std::string &id, RsWikiGroup &group)
+{
+	void *groupData = NULL;
+	RsGroupMetaData meta;
+	if (getGroupData(id, groupData) && getGroupSummary(id, meta))
+	{
+		RsWikiGroup *pG = (RsWikiGroup *) groupData;
+		group = *pG;
+
+		// update definitive version of the metadata.
+		group.mMeta = meta;
+
+		std::cerr << "WikiDataProxy::getGroup() Id: " << id;
+		std::cerr << " MetaData: " << meta << " DataPointer: " << groupData;
+		std::cerr << std::endl;
+		return true;
+	}
+
+	std::cerr << "WikiDataProxy::getGroup() FAILED Id: " << id;
+	std::cerr << std::endl;
+
+	return false;
+}
+
+bool WikiDataProxy::getPage(const std::string &id, RsWikiPage &page)
+{
+	void *msgData = NULL;
+	RsMsgMetaData meta;
+	if (getMsgData(id, msgData) && getMsgSummary(id, meta))
+	{
+		RsWikiPage *pP = (RsWikiPage *) msgData;
+		// Shallow copy of thumbnail.
+		page = *pP;
+	
+		// update definitive version of the metadata.
+		page.mMeta = meta;
+
+		std::cerr << "WikiDataProxy::getPage() Id: " << id;
+		std::cerr << " MetaData: " << meta << " DataPointer: " << msgData;
+		std::cerr << std::endl;
+		return true;
+	}
+
+	std::cerr << "WikiDataProxy::getPage() FAILED Id: " << id;
+	std::cerr << std::endl;
+
+	return false;
+}
+
+bool WikiDataProxy::addGroup(const RsWikiGroup &group)
+{
+	// Make duplicate.
+	RsWikiGroup *pG = new RsWikiGroup();
+	*pG = group;
+
+	std::cerr << "WikiDataProxy::addGroup()";
+	std::cerr << " MetaData: " << pG->mMeta << " DataPointer: " << pG;
+	std::cerr << std::endl;
+
+	return createGroup(pG);
+}
+
+
+bool WikiDataProxy::addPage(const RsWikiPage &page)
+{
+	// Make duplicate.
+	RsWikiPage *pP = new RsWikiPage();
+	*pP = page;
+
+	std::cerr << "WikiDataProxy::addPage()";
+	std::cerr << " MetaData: " << pP->mMeta << " DataPointer: " << pP;
+	std::cerr << std::endl;
+
+	return createMsg(pP);
+}
+
+
+
+        /* These Functions must be overloaded to complete the service */
+bool WikiDataProxy::convertGroupToMetaData(void *groupData, RsGroupMetaData &meta)
+{
+	RsWikiGroup *group = (RsWikiGroup *) groupData;
+	meta = group->mMeta;
+
+	return true;
+}
+
+bool WikiDataProxy::convertMsgToMetaData(void *msgData, RsMsgMetaData &meta)
+{
+	RsWikiPage *page = (RsWikiPage *) msgData;
+	meta = page->mMeta;
+
+	return true;
+}

libretroshare/src/services/p3wikiservice.h

@@ -0,0 +1,129 @@
+/*
+ * libretroshare/src/services: p3wikiservice.h
+ *
+ * Wiki interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef P3_WIKI_SERVICE_HEADER
+#define P3_WIKI_SERVICE_HEADER
+
+#include "services/p3gxsservice.h"
+#include "retroshare/rswiki.h"
+
+#include <map>
+#include <string>
+
+/* 
+ * Wiki Service
+ *
+ * This is an example service for the new cache system.
+ * For the moment, it will only hold data passed to it from the GUI.
+ * and spew that back when asked....
+ *
+ * We are doing it like this - so we can check the required interface functionality.
+ *
+ * Expect it won't take long before it'll be properly linked into the backend!
+ *
+ * This will be transformed into a Plugin Service, once the basics have been worked out.
+ *
+ */
+
+class WikiDataProxy: public GxsDataProxy
+{
+	public:
+
+	bool getGroup(const std::string &id, RsWikiGroup &group);
+	bool getPage(const std::string &id, RsWikiPage &wiki);
+
+	bool addGroup(const RsWikiGroup &group);
+	bool addPage(const RsWikiPage &page);
+
+        /* These Functions must be overloaded to complete the service */
+virtual bool convertGroupToMetaData(void *groupData, RsGroupMetaData &meta);
+virtual bool convertMsgToMetaData(void *msgData, RsMsgMetaData &meta);
+};
+
+
+class p3WikiService: public p3GxsDataService, public RsWiki
+{
+	public:
+
+	p3WikiService(uint16_t type);
+
+virtual int	tick();
+
+	public:
+
+
+virtual bool updated();
+
+       /* Data Requests */
+virtual bool requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds);
+
+        /* Generic Lists */
+virtual bool getGroupList(         const uint32_t &token, std::list<std::string> &groupIds);
+virtual bool getMsgList(           const uint32_t &token, std::list<std::string> &msgIds);
+
+        /* Generic Summary */
+virtual bool getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo);
+virtual bool getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo);
+
+        /* Actual Data -> specific to Interface */
+        /* Specific Service Data */
+virtual bool getGroupData(const uint32_t &token, RsWikiGroup &group);
+virtual bool getMsgData(const uint32_t &token, RsWikiPage &page);
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token);
+
+        /* Cancel Request */
+virtual bool cancelRequest(const uint32_t &token);
+
+        //////////////////////////////////////////////////////////////////////////////
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str);
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str);
+
+virtual bool groupRestoreKeys(const std::string &groupId);
+virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers);
+
+virtual bool createGroup(uint32_t &token, RsWikiGroup &group, bool isNew);
+virtual bool createPage(uint32_t &token, RsWikiPage &page, bool isNew);
+
+	private:
+
+std::string genRandomId();
+
+	WikiDataProxy *mWikiProxy;
+
+	RsMutex mWikiMtx;
+
+	bool mUpdated;
+
+
+};
+
+#endif 

libretroshare/src/services/p3wire.cc

@@ -0,0 +1,598 @@
+/*
+ * libretroshare/src/services p3wire.cc
+ *
+ * Wire interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "services/p3wire.h"
+
+#include "util/rsrandom.h"
+
+/****
+ * #define WIKI_DEBUG 1
+ ****/
+
+RsWire *rsWire = NULL;
+
+
+/********************************************************************************/
+/******************* Startup / Tick    ******************************************/
+/********************************************************************************/
+
+p3Wire::p3Wire(uint16_t type)
+	:p3GxsDataService(type, new WireDataProxy()), mWireMtx("p3Wire"), mUpdated(true)
+{
+     	RsStackMutex stack(mWireMtx); /********** STACK LOCKED MTX ******/
+
+	mWireProxy = (WireDataProxy *) mProxy;
+	return;
+}
+
+
+int	p3Wire::tick()
+{
+	std::cerr << "p3Wire::tick()";
+	std::cerr << std::endl;
+
+	fakeprocessrequests();
+	
+	return 0;
+}
+
+bool p3Wire::updated()
+{
+	RsStackMutex stack(mWireMtx); /********** STACK LOCKED MTX ******/
+
+	if (mUpdated)
+	{
+		mUpdated = false;
+		return true;
+	}
+	return false;
+}
+
+
+
+       /* Data Requests */
+bool p3Wire::requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds)
+{
+	generateToken(token);
+	std::cerr << "p3Wire::requestGroupInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_GROUPS, groupIds);
+
+	return true;
+}
+
+bool p3Wire::requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds)
+{
+	generateToken(token);
+	std::cerr << "p3Wire::requestMsgInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGS, groupIds);
+
+	return true;
+}
+
+bool p3Wire::requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds)
+{
+	generateToken(token);
+	std::cerr << "p3Wire::requestMsgRelatedInfo() gets Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGRELATED, msgIds);
+
+	return true;
+}
+
+        /* Generic Lists */
+bool p3Wire::getGroupList(         const uint32_t &token, std::list<std::string> &groupIds)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_LIST)
+	{
+		std::cerr << "p3Wire::getGroupList() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3Wire::getGroupList() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3Wire::getGroupList() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	bool ans = loadRequestOutList(token, groupIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	return ans;
+}
+
+
+
+
+bool p3Wire::getMsgList(           const uint32_t &token, std::list<std::string> &msgIds)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_LIST)
+	{
+		std::cerr << "p3Wire::getMsgList() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3Wire::getMsgList() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3Wire::getMsgList() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	bool ans = loadRequestOutList(token, msgIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	return ans;
+}
+
+
+        /* Generic Summary */
+bool p3Wire::getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_SUMMARY)
+	{
+		std::cerr << "p3Wire::getGroupSummary() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3Wire::getGroupSummary() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3Wire::getGroupSummary() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	std::list<std::string> groupIds;
+	bool ans = loadRequestOutList(token, groupIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	/* convert to RsGroupMetaData */
+	mProxy->getGroupSummary(groupIds, groupInfo);
+
+	return ans;
+}
+
+bool p3Wire::getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	if (anstype != RS_TOKREQ_ANSTYPE_SUMMARY)
+	{
+		std::cerr << "p3Wire::getMsgSummary() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3Wire::getMsgSummary() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3Wire::getMsgSummary() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+
+	std::list<std::string> msgIds;
+	bool ans = loadRequestOutList(token, msgIds);	
+	updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+
+	/* convert to RsMsgMetaData */
+	mProxy->getMsgSummary(msgIds, msgInfo);
+
+	return ans;
+}
+
+
+        /* Specific Service Data */
+bool p3Wire::getGroupData(const uint32_t &token, RsWireGroup &group)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+	
+
+	if (anstype != RS_TOKREQ_ANSTYPE_DATA)
+	{
+		std::cerr << "p3Wire::getGroupData() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (reqtype != GXS_REQUEST_TYPE_GROUPS)
+	{
+		std::cerr << "p3Wire::getGroupData() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3Wire::getGroupData() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+	
+	std::string id;
+	if (!popRequestOutList(token, id))
+	{
+		/* finished */
+		updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		return false;
+	}
+	
+	/* convert to RsWireGroup */
+	bool ans = mWireProxy->getGroup(id, group);
+	return ans;
+}
+
+
+bool p3Wire::getMsgData(const uint32_t &token, RsWirePulse &pulse)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+	
+
+	if (anstype != RS_TOKREQ_ANSTYPE_DATA)
+	{
+		std::cerr << "p3Wire::getMsgData() ERROR AnsType Wrong" << std::endl;
+		return false;
+	}
+	
+	if ((reqtype != GXS_REQUEST_TYPE_MSGS) && (reqtype != GXS_REQUEST_TYPE_MSGRELATED))
+	{
+		std::cerr << "p3Wire::getMsgData() ERROR ReqType Wrong" << std::endl;
+		return false;
+	}
+	
+	if (status != GXS_REQUEST_STATUS_COMPLETE)
+	{
+		std::cerr << "p3Wire::getMsgData() ERROR Status Incomplete" << std::endl;
+		return false;
+	}
+	
+	std::string id;
+	if (!popRequestOutList(token, id))
+	{
+		/* finished */
+		updateRequestStatus(token, GXS_REQUEST_STATUS_DONE);
+		return false;
+	}
+	
+	/* convert to RsWirePulse */
+	bool ans = mWireProxy->getPulse(id, pulse);
+	return ans;
+}
+
+
+
+        /* Poll */
+uint32_t p3Wire::requestStatus(const uint32_t token)
+{
+	uint32_t status;
+	uint32_t reqtype;
+	uint32_t anstype;
+	time_t ts;
+	checkRequestStatus(token, status, reqtype, anstype, ts);
+
+	return status;
+}
+
+
+        /* Cancel Request */
+bool p3Wire::cancelRequest(const uint32_t &token)
+{
+	return clearRequest(token);
+}
+
+        //////////////////////////////////////////////////////////////////////////////
+
+bool p3Wire::setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask)
+{
+        return mWireProxy->setMessageStatus(msgId, status, statusMask);
+}
+
+
+bool p3Wire::setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask)
+{
+        return mWireProxy->setGroupStatus(groupId, status, statusMask);
+}
+
+bool p3Wire::setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask)
+{
+	return mWireProxy->setGroupSubscribeFlags(groupId, subscribeFlags, subscribeMask);
+}
+
+bool p3Wire::setMessageServiceString(const std::string &msgId, const std::string &str)
+{
+        return mWireProxy->setMessageServiceString(msgId, str);
+}
+
+bool p3Wire::setGroupServiceString(const std::string &grpId, const std::string &str)
+{
+        return mWireProxy->setGroupServiceString(grpId, str);
+}
+
+
+bool p3Wire::groupRestoreKeys(const std::string &groupId)
+{
+	return false;
+}
+
+bool p3Wire::groupShareKeys(const std::string &groupId, std::list<std::string>& peers)
+{
+	return false;
+}
+
+
+/********************************************************************************************/
+
+	
+std::string p3Wire::genRandomId()
+{
+	std::string randomId;
+	for(int i = 0; i < 20; i++)
+	{
+		randomId += (char) ('a' + (RSRandom::random_u32() % 26));
+	}
+	
+	return randomId;
+}
+	
+
+bool p3Wire::createGroup(uint32_t &token, RsWireGroup &group, bool isNew)
+{
+	if (group.mMeta.mGroupId.empty())
+	{
+		/* new photo */
+
+		/* generate a temp id */
+		group.mMeta.mGroupId = genRandomId();
+	}
+	else
+	{
+		std::cerr << "p3Wire::createGroup() Group with existing Id... dropping";
+		std::cerr << std::endl;
+		return false;
+	}
+
+	{	
+		RsStackMutex stack(mWireMtx); /********** STACK LOCKED MTX ******/
+
+		mUpdated = true;
+		mWireProxy->addGroup(group);
+	}
+	
+	// Fake a request to return the GroupMetaData.
+	generateToken(token);
+	uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY;
+	RsTokReqOptions opts; // NULL is good.
+	std::list<std::string> groupIds;
+	groupIds.push_back(group.mMeta.mGroupId); // It will just return this one.
+	
+	std::cerr << "p3Wiree::createGroup() Generating Request Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_GROUPS, groupIds);
+	
+	return true;
+}
+
+
+
+
+bool p3Wire::createPulse(uint32_t &token, RsWirePulse &pulse, bool isNew)
+{
+	if (pulse.mMeta.mGroupId.empty())
+	{
+		/* new photo */
+		std::cerr << "p3Wire::createPulse() Missing PulseID";
+		std::cerr << std::endl;
+		return false;
+	}
+	
+	/* check if its a mod or new pulse */
+	if (pulse.mMeta.mOrigMsgId.empty())
+	{
+		std::cerr << "p3Wire::createPulse() New Pulse";
+		std::cerr << std::endl;
+
+		/* new pulse, generate a new OrigPulseId */
+		pulse.mMeta.mOrigMsgId = genRandomId();
+		pulse.mMeta.mMsgId = pulse.mMeta.mOrigMsgId;
+	}
+	else
+	{
+		std::cerr << "p3Wire::createPulse() Modified Pulse";
+		std::cerr << std::endl;
+
+		/* mod pulse, keep orig pulse id, generate a new PulseId */
+		pulse.mMeta.mMsgId = genRandomId();
+	}
+
+	std::cerr << "p3Wire::createPulse() GroupId: " << pulse.mMeta.mGroupId;
+	std::cerr << std::endl;
+	std::cerr << "p3Wire::createPulse() PulseId: " << pulse.mMeta.mMsgId;
+	std::cerr << std::endl;
+	std::cerr << "p3Wire::createPulse() OrigPulseId: " << pulse.mMeta.mOrigMsgId;
+	std::cerr << std::endl;
+
+	{
+		RsStackMutex stack(mWireMtx); /********** STACK LOCKED MTX ******/
+
+		mUpdated = true;
+		mWireProxy->addPulse(pulse);
+	}
+	
+	// Fake a request to return the MsgMetaData.
+	generateToken(token);
+	uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY;
+	RsTokReqOptions opts; // NULL is good.
+	std::list<std::string> msgIds;
+	msgIds.push_back(pulse.mMeta.mMsgId); // It will just return this one.
+	
+	std::cerr << "p3Wire::createPulse() Generating Request Token: " << token << std::endl;
+	storeRequest(token, ansType, opts, GXS_REQUEST_TYPE_MSGRELATED, msgIds);
+
+	return true;
+}
+
+
+
+/********************************************************************************************/
+
+
+	
+bool WireDataProxy::getGroup(const std::string &id, RsWireGroup &group)
+{
+	void *groupData = NULL;
+	RsGroupMetaData meta;
+	if (getGroupData(id, groupData) && getGroupSummary(id, meta))
+	{
+		RsWireGroup *pG = (RsWireGroup *) groupData;
+		group = *pG;
+
+		// update definitive version of the metadata.
+		group.mMeta = meta;
+
+		std::cerr << "WireDataProxy::getGroup() Id: " << id;
+		std::cerr << " MetaData: " << meta << " DataPointer: " << groupData;
+		std::cerr << std::endl;
+		return true;
+	}
+
+	std::cerr << "WireDataProxy::getGroup() FAILED Id: " << id;
+	std::cerr << std::endl;
+
+	return false;
+}
+
+bool WireDataProxy::getPulse(const std::string &id, RsWirePulse &pulse)
+{
+	void *msgData = NULL;
+	RsMsgMetaData meta;
+	if (getMsgData(id, msgData) && getMsgSummary(id, meta))
+	{
+		RsWirePulse *pP = (RsWirePulse *) msgData;
+		// Shallow copy of thumbnail.
+		pulse = *pP;
+	
+		// update definitive version of the metadata.
+		pulse.mMeta = meta;
+
+		std::cerr << "WireDataProxy::getPulse() Id: " << id;
+		std::cerr << " MetaData: " << meta << " DataPointer: " << msgData;
+		std::cerr << std::endl;
+		return true;
+	}
+
+	std::cerr << "WireDataProxy::getPulse() FAILED Id: " << id;
+	std::cerr << std::endl;
+
+	return false;
+}
+
+bool WireDataProxy::addGroup(const RsWireGroup &group)
+{
+	// Make duplicate.
+	RsWireGroup *pG = new RsWireGroup();
+	*pG = group;
+
+	std::cerr << "WireDataProxy::addGroup()";
+	std::cerr << " MetaData: " << pG->mMeta << " DataPointer: " << pG;
+	std::cerr << std::endl;
+
+	return createGroup(pG);
+}
+
+
+bool WireDataProxy::addPulse(const RsWirePulse &pulse)
+{
+	// Make duplicate.
+	RsWirePulse *pP = new RsWirePulse();
+	*pP = pulse;
+
+	std::cerr << "WireDataProxy::addPulse()";
+	std::cerr << " MetaData: " << pP->mMeta << " DataPointer: " << pP;
+	std::cerr << std::endl;
+
+	return createMsg(pP);
+}
+
+
+
+        /* These Functions must be overloaded to complete the service */
+bool WireDataProxy::convertGroupToMetaData(void *groupData, RsGroupMetaData &meta)
+{
+	RsWireGroup *group = (RsWireGroup *) groupData;
+	meta = group->mMeta;
+
+	return true;
+}
+
+bool WireDataProxy::convertMsgToMetaData(void *msgData, RsMsgMetaData &meta)
+{
+	RsWirePulse *page = (RsWirePulse *) msgData;
+	meta = page->mMeta;
+
+	return true;
+}

libretroshare/src/services/p3wire.h

@@ -0,0 +1,120 @@
+/*
+ * libretroshare/src/services: p3wire.h
+ *
+ * Wire interface for RetroShare.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef P3_WIRE_SERVICE_HEADER
+#define P3_WIRE_SERVICE_HEADER
+
+#include "services/p3gxsservice.h"
+
+#include "retroshare/rswire.h"
+
+#include <map>
+#include <string>
+
+/* 
+ * Wire Service
+ *
+ */
+class WireDataProxy: public GxsDataProxy
+{
+        public:
+
+        bool getGroup(const std::string &id, RsWireGroup &group);
+        bool getPulse(const std::string &id, RsWirePulse &pulse);
+
+        bool addGroup(const RsWireGroup &group);
+        bool addPulse(const RsWirePulse &pulse);
+
+        /* These Functions must be overloaded to complete the service */
+virtual bool convertGroupToMetaData(void *groupData, RsGroupMetaData &meta);
+virtual bool convertMsgToMetaData(void *msgData, RsMsgMetaData &meta);
+};
+
+
+class p3Wire: public p3GxsDataService, public RsWire
+{
+	public:
+
+	p3Wire(uint16_t type);
+
+virtual int	tick();
+
+	public:
+
+
+virtual bool updated();
+
+       /* Data Requests */
+virtual bool requestGroupInfo(     uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgInfo(       uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &groupIds);
+virtual bool requestMsgRelatedInfo(uint32_t &token, uint32_t ansType, const RsTokReqOptions &opts, const std::list<std::string> &msgIds);
+
+        /* Generic Lists */
+virtual bool getGroupList(         const uint32_t &token, std::list<std::string> &groupIds);
+virtual bool getMsgList(           const uint32_t &token, std::list<std::string> &msgIds);
+
+        /* Generic Summary */
+virtual bool getGroupSummary(      const uint32_t &token, std::list<RsGroupMetaData> &groupInfo);
+virtual bool getMsgSummary(        const uint32_t &token, std::list<RsMsgMetaData> &msgInfo);
+
+        /* Actual Data -> specific to Interface */
+        /* Specific Service Data */
+virtual bool getGroupData(const uint32_t &token, RsWireGroup &group);
+virtual bool getMsgData(const uint32_t &token, RsWirePulse &page);
+
+        /* Poll */
+virtual uint32_t requestStatus(const uint32_t token);
+
+        /* Cancel Request */
+virtual bool cancelRequest(const uint32_t &token);
+
+        //////////////////////////////////////////////////////////////////////////////
+virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupStatus(const std::string &groupId, const uint32_t status, const uint32_t statusMask);
+virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
+virtual bool setMessageServiceString(const std::string &msgId, const std::string &str);
+virtual bool setGroupServiceString(const std::string &grpId, const std::string &str);
+
+virtual bool groupRestoreKeys(const std::string &groupId);
+virtual bool groupShareKeys(const std::string &groupId, std::list<std::string>& peers);
+
+virtual bool createGroup(uint32_t &token, RsWireGroup &group, bool isNew);
+virtual bool createPulse(uint32_t &token, RsWirePulse &pulse, bool isNew);
+
+	private:
+
+std::string genRandomId();
+
+	WireDataProxy *mWireProxy;
+
+	RsMutex mWireMtx;
+
+	/***** below here is locked *****/
+
+	bool mUpdated;
+
+};
+
+#endif 

libretroshare/src/tests/gxs/data_support.cc

@@ -0,0 +1,97 @@
+#include "support.h"
+#include "data_support.h"
+
+
+bool operator==(const RsNxsGrp& l, const RsNxsGrp& r){
+
+    if(l.grpId != r.grpId) return false;
+    if(!(l.grp == r.grp) ) return false;
+    if(!(l.meta == r.meta) ) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+bool operator==(const RsNxsMsg& l, const RsNxsMsg& r){
+
+
+    if(l.msgId != r.msgId) return false;
+    if(l.grpId != r.grpId) return false;
+    if(! (l.msg == r.msg) ) return false;
+    if(! (l.meta == r.meta) ) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+
+void init_item(RsNxsGrp& nxg)
+{
+
+    nxg.clear();
+
+    randString(SHORT_STR, nxg.grpId);
+    nxg.transactionNumber = rand()%23;
+    init_item(nxg.grp);
+    init_item(nxg.meta);
+    return;
+}
+
+
+
+
+
+void init_item(RsNxsMsg& nxm)
+{
+    nxm.clear();
+
+    randString(SHORT_STR, nxm.msgId);
+    randString(SHORT_STR, nxm.grpId);
+    init_item(nxm.msg);
+    init_item(nxm.meta);
+    nxm.transactionNumber = rand()%23;
+
+    return;
+}
+
+void init_item(RsGxsGrpMetaData* metaGrp)
+{
+
+    randString(SHORT_STR, metaGrp->mGroupId);
+    randString(SHORT_STR, metaGrp->mOrigGrpId);
+    randString(SHORT_STR, metaGrp->mAuthorId);
+    randString(SHORT_STR, metaGrp->mGroupName);
+
+    init_item(metaGrp->adminSign);
+    init_item(metaGrp->keys);
+    init_item(metaGrp->idSign);
+
+    metaGrp->mPublishTs = rand()%3452;
+    metaGrp->mGroupFlags = rand()%43;
+
+    metaGrp->mGroupStatus = rand()%313;
+    metaGrp->mSubscribeFlags = rand()%2251;
+    metaGrp->mMsgCount = rand()%2421;
+    metaGrp->mLastPost = rand()%2211;
+    metaGrp->mPop = rand()%5262;
+}
+
+void init_item(RsGxsMsgMetaData* metaMsg)
+{
+
+    randString(SHORT_STR, metaMsg->mGroupId);
+    randString(SHORT_STR, metaMsg->mMsgId);
+    randString(SHORT_STR, metaMsg->mThreadId);
+    randString(SHORT_STR, metaMsg->mParentId);
+    randString(SHORT_STR, metaMsg->mAuthorId);
+    randString(SHORT_STR, metaMsg->mOrigMsgId);
+    randString(SHORT_STR, metaMsg->mMsgName);
+
+    init_item(metaMsg->pubSign);
+    init_item(metaMsg->idSign);
+
+    metaMsg->mPublishTs = rand()%313;
+    metaMsg->mMsgFlags = rand()%224;
+    metaMsg->mMsgStatus = rand()%4242;
+    metaMsg->mChildTs = rand()%221;
+}

libretroshare/src/tests/gxs/data_support.h

@@ -0,0 +1,15 @@
+#ifndef DATA_SUPPORT_H
+#define DATA_SUPPORT_H
+
+#include "serialiser/rsnxsitems.h"
+#include "gxs/rsgxsdata.h"
+
+bool operator==(const RsNxsGrp&, const RsNxsGrp&);
+bool operator==(const RsNxsMsg&, const RsNxsMsg&);
+
+void init_item(RsNxsGrp& nxg);
+void init_item(RsNxsMsg& nxm);
+void init_item(RsGxsGrpMetaData* metaGrp);
+void init_item(RsGxsMsgMetaData* metaMsg);
+
+#endif // DATA_SUPPORT_H

libretroshare/src/tests/gxs/nxstesthub.cc

@@ -0,0 +1,81 @@
+#include "nxstesthub.h"
+
+NxsTestHub::NxsTestHub(NxsTestScenario* nts) : mTestScenario(nts)
+{
+
+	netServicePairs.first = new RsGxsNetService(mTestScenario->getServiceType(),
+			mTestScenario->dummyDataService1(), &netMgr1, mTestScenario);
+	netServicePairs.second = new RsGxsNetService(mTestScenario->getServiceType(),
+			mTestScenario->dummyDataService2(), &netMgr2, mTestScenario);
+
+	mServicePairs.first = netServicePairs.first;
+	mServicePairs.second = netServicePairs.second;
+
+	createThread(*(netServicePairs.first));
+	createThread(*(netServicePairs.second));
+}
+
+NxsTestHub::~NxsTestHub()
+{
+	delete netServicePairs.first;
+	delete netServicePairs.second;
+}
+
+
+void NxsTestHub::run()
+{
+
+	std::list<RsItem*> send_queue_s1, send_queue_s2;
+
+	while(isRunning()){
+
+		// make thread sleep for a couple secs
+		usleep(3000);
+
+		p3Service* s1 = mServicePairs.first;
+		p3Service* s2 = mServicePairs.second;
+
+		RsItem* item = NULL;
+		while((item = s1->send()) != NULL)
+		{
+			item->PeerId("PeerB");
+			send_queue_s1.push_back(item);
+		}
+
+		while((item = s2->send()) != NULL)
+		{
+			item->PeerId("PeerA");
+			send_queue_s2.push_back(item);
+		}
+
+		while(!send_queue_s1.empty()){
+			item = send_queue_s1.front();
+			s2->receive(dynamic_cast<RsRawItem*>(item));
+			send_queue_s1.pop_front();
+		}
+
+		while(!send_queue_s2.empty()){
+			item = send_queue_s2.front();
+			s1->receive(dynamic_cast<RsRawItem*>(item));
+			send_queue_s2.pop_front();
+		}
+
+		// tick services so nxs net services process items
+		s1->tick();
+		s2->tick();
+	}
+
+	// also shut down this net service peers if this goes down
+	netServicePairs.first->join();
+	netServicePairs.second->join();
+}
+
+void NxsTestHub::cleanUp()
+{
+	mTestScenario->cleanUp();
+}
+
+bool NxsTestHub::testsPassed()
+{
+	return false;
+}

libretroshare/src/tests/gxs/nxstesthub.h

@@ -0,0 +1,109 @@
+#ifndef NXSTESTHUB_H
+#define NXSTESTHUB_H
+
+#include "util/rsthreads.h"
+#include "gxs/rsgxsnetservice.h"
+#include "nxstestscenario.h"
+
+// it would probably be useful if the test scenario
+// provided the net dummy managers
+// hence one could envision synchronising between an arbitrary number
+// of peers
+
+
+class NxsNetDummyMgr1 : public RsNxsNetMgr
+{
+
+public:
+
+	NxsNetDummyMgr1() : mOwnId("peerA") {
+
+		mPeers.insert("peerB");
+	}
+
+    std::string getOwnId()  { return mOwnId; }
+    void getOnlineList(std::set<std::string>& ssl_peers) { ssl_peers = mPeers; }
+
+private:
+
+    std::string mOwnId;
+    std::set<std::string> mPeers;
+
+};
+
+class NxsNetDummyMgr2 : public RsNxsNetMgr
+{
+
+public:
+
+	NxsNetDummyMgr2() : mOwnId("peerB") {
+
+		mPeers.insert("peerA");
+
+	}
+
+    std::string getOwnId()  { return mOwnId; }
+    void getOnlineList(std::set<std::string>& ssl_peers) { ssl_peers = mPeers; }
+
+private:
+
+    std::string mOwnId;
+    std::set<std::string> mPeers;
+};
+
+
+/*!
+ * Testing of nxs services occurs through use of two services
+ * When a service sends this class can interrogate the send and the receives of
+ *
+ * NxsScenario stores the type of synchronisation to be tested
+ * Operation:
+ * First NxsTestHub needs to be instantiated with a test scenario
+ *   * The scenario contains two databases to be used on the communicating pair of RsGxsNetService instances (net instances)
+ * The Test hub has a ticker service for the p3Services which allows the netservices to search what groups and messages they have
+ * and synchronise according to their subscriptions. The default is to subscribe to all groups held by other peer
+ * The threads for both net instances are started which begins their processing of transactions
+ */
+class NxsTestHub : public RsThread
+{
+public:
+
+
+	/*!
+	 * This construct the test hub
+	 * for a give scenario in mind
+	 */
+    NxsTestHub(NxsTestScenario*);
+
+
+    /*!
+     *
+     */
+    virtual ~NxsTestHub();
+
+    /*!
+     * To be called only after this thread has
+     * been shutdown
+     */
+    bool testsPassed();
+
+    /*!
+     *  This simulates the p3Service ticker and calls both gxs net services tick methods
+     *  Also enables transport of messages between both services
+     */
+    void run();
+
+
+    void cleanUp();
+private:
+
+    std::pair<p3Service*, p3Service*> mServicePairs;
+    std::pair<RsGxsNetService*, RsGxsNetService*> netServicePairs;
+    NxsTestScenario *mTestScenario;
+
+    NxsNetDummyMgr1 netMgr1;
+    NxsNetDummyMgr2 netMgr2;
+
+};
+
+#endif // NXSTESTHUB_H

libretroshare/src/tests/gxs/nxstestscenario.cc

@@ -0,0 +1,161 @@
+/*
+ * nxstestscenario.cc
+ *
+ *  Created on: 10 Jul 2012
+ *      Author: crispy
+ */
+
+#include "nxstestscenario.h"
+#include "gxs/rsdataservice.h"
+#include "data_support.h"
+
+NxsMessageTest::NxsMessageTest(uint16_t servtype)
+: mServType(servtype)
+{
+	mStorePair.first = new RsDataService(".", "dStore1", mServType);
+	mStorePair.second = new RsDataService(".", "dStore2", mServType);
+
+	setUpDataBases();
+}
+
+std::string NxsMessageTest::getTestName()
+{
+	return std::string("Nxs Message Test!");
+}
+
+NxsMessageTest::~NxsMessageTest(){
+	delete mStorePair.first;
+	delete mStorePair.second;
+}
+void NxsMessageTest::setUpDataBases()
+{
+	// create several groups and then messages of that
+	// group for both second and first of pair
+	RsDataService* dStore = dynamic_cast<RsDataService*>(mStorePair.first);
+	populateStore(dStore);
+
+	dStore = dynamic_cast<RsDataService*>(mStorePair.second);
+	populateStore(dStore);
+
+	dStore = NULL;
+	return;
+}
+
+uint16_t NxsMessageTest::getServiceType()
+{
+	return mServType;
+}
+
+void NxsMessageTest::populateStore(RsGeneralDataService* dStore)
+{
+
+	int nGrp = rand()%7;
+	std::vector<std::string> grpIdList;
+	std::map<RsNxsGrp*, RsGxsGrpMetaData*> grps;
+	RsNxsGrp* grp = NULL;
+	RsGxsGrpMetaData* grpMeta =NULL;
+	for(int i = 0; i < nGrp; i++)
+	{
+		std::pair<RsNxsGrp*, RsGxsGrpMetaData*> p;
+	   grp = new RsNxsGrp(mServType);
+	   grpMeta = new RsGxsGrpMetaData();
+	   p.first = grp;
+	   p.second = grpMeta;
+	   init_item(*grp);
+	   init_item(grpMeta);
+	   grpMeta->mGroupId = grp->grpId;
+	   grps.insert(p);
+	   grpIdList.push_back(grp->grpId);
+	   grpMeta = NULL;
+	   grp = NULL;
+	}
+
+	dStore->storeGroup(grps);
+
+
+	std::map<RsNxsGrp*, RsGxsGrpMetaData*>::iterator grp_it
+	= grps.begin();
+    for(; grp_it != grps.end(); grp_it++)
+	{
+		delete grp_it->first;
+		delete grp_it->second;
+	}
+
+
+	int nMsgs = rand()%23;
+	std::map<RsNxsMsg*, RsGxsMsgMetaData*> msgs;
+	RsNxsMsg* msg = NULL;
+	RsGxsMsgMetaData* msgMeta = NULL;
+
+    for(int i=0; i<nMsgs; i++)
+    {
+        msg = new RsNxsMsg(mServType);
+        msgMeta = new RsGxsMsgMetaData();
+        init_item(*msg);
+        init_item(msgMeta);
+        std::pair<RsNxsMsg*, RsGxsMsgMetaData*> p(msg, msgMeta);
+
+        // pick a grp at random to associate the msg to
+        const std::string& grpId = grpIdList[rand()%nGrp];
+        msgMeta->mMsgId = msg->msgId;
+        msgMeta->mGroupId = msg->grpId = grpId;
+
+        msg = NULL;
+        msgMeta = NULL;
+
+        msgs.insert(p);
+    }
+
+
+    dStore->storeMessage(msgs);
+
+    // clean up
+    std::map<RsNxsMsg*, RsGxsMsgMetaData*>::iterator msg_it
+    = msgs.begin();
+
+    for(; msg_it != msgs.end(); msg_it++)
+    {
+    	delete msg_it->first;
+    	delete msg_it->second;
+    }
+
+    return;
+}
+
+void NxsMessageTest::notifyNewMessages(std::vector<RsNxsMsg*>& messages)
+{
+	std::vector<RsNxsMsg*>::iterator vit = messages.begin();
+
+	for(; vit != messages.end(); vit++)
+	{
+		mPeerMsgs[(*vit)->PeerId()].push_back(*vit);
+	}
+}
+
+void NxsMessageTest::notifyNewGroups(std::vector<RsNxsGrp*>& groups)
+{
+	std::vector<RsNxsGrp*>::iterator vit = groups.begin();
+
+	for(; vit != groups.end(); vit++)
+	{
+		mPeerGrps[(*vit)->PeerId()].push_back(*vit);
+	}
+}
+
+RsGeneralDataService* NxsMessageTest::dummyDataService1()
+{
+	return mStorePair.first;
+}
+
+RsGeneralDataService* NxsMessageTest::dummyDataService2()
+{
+	return mStorePair.second;
+}
+
+void NxsMessageTest::cleanUp()
+{
+	mStorePair.first->resetDataStore();
+	mStorePair.second->resetDataStore();
+
+	return;
+}

libretroshare/src/tests/gxs/nxstestscenario.h

@@ -0,0 +1,83 @@
+/*
+ * nxstestscenario.h
+ *
+ *  Created on: 10 Jul 2012
+ *      Author: crispy
+ */
+
+#ifndef NXSTESTSCENARIO_H_
+#define NXSTESTSCENARIO_H_
+
+#include <map>
+#include "gxs/rsdataservice.h"
+#include "gxs/rsnxsobserver.h"
+
+/*!
+ * This scenario module provides data resources
+ */
+class NxsTestScenario : public RsNxsObserver
+{
+
+public:
+
+    virtual std::string getTestName() = 0;
+    virtual RsGeneralDataService* dummyDataService1() = 0;
+    virtual RsGeneralDataService* dummyDataService2() = 0;
+    virtual uint16_t getServiceType() = 0;
+
+    /*!
+     * Call to remove files created
+     * in the test directory
+     */
+    virtual void cleanUp() = 0;
+
+
+};
+
+class NxsMessageTest : public NxsTestScenario
+{
+
+public:
+
+	NxsMessageTest(uint16_t servtype);
+	virtual ~NxsMessageTest();
+    std::string getTestName();
+    uint16_t getServiceType();
+    RsGeneralDataService* dummyDataService1();
+    RsGeneralDataService* dummyDataService2();
+
+    /*!
+     * Call to remove files created
+     * in the test directory
+     */
+    void cleanUp();
+
+public:
+
+    /*!
+     * @param messages messages are deleted after function returns
+     */
+    void notifyNewMessages(std::vector<RsNxsMsg*>& messages);
+
+    /*!
+     * @param messages messages are deleted after function returns
+     */
+    void notifyNewGroups(std::vector<RsNxsGrp*>& groups);
+
+
+private:
+    void setUpDataBases();
+    void populateStore(RsGeneralDataService* dStore);
+
+private:
+
+    std::string mTestName;
+    std::pair<RsGeneralDataService*, RsGeneralDataService*> mStorePair;
+    std::map<std::string, std::vector<RsNxsMsg*> > mPeerMsgs;
+    std::map<std::string, std::vector<RsNxsGrp*> > mPeerGrps;
+    uint16_t mServType;
+
+};
+
+
+#endif /* NXSTESTSCENARIO_H_ */

libretroshare/src/tests/gxs/rs_test.pro

@@ -0,0 +1,73 @@
+#-------------------------------------------------
+#
+# Project created by QtCreator 2012-05-06T09:19:26
+#
+#-------------------------------------------------
+
+QT       += core network
+
+QT       -= gui
+
+TARGET = rs_test
+CONFIG   += console
+CONFIG   -= app_bundle
+
+TEMPLATE = app
+
+win32 {
+        # Switch on extra warnings
+        QMAKE_CFLAGS += -Wextra
+        QMAKE_CXXFLAGS += -Wextra
+
+        # Switch off optimization for release version
+        QMAKE_CXXFLAGS_RELEASE -= -O2
+        QMAKE_CXXFLAGS_RELEASE += -O0
+        QMAKE_CFLAGS_RELEASE -= -O2
+        QMAKE_CFLAGS_RELEASE += -O0
+
+        # Switch on optimization for debug version
+        #QMAKE_CXXFLAGS_DEBUG += -O2
+        #QMAKE_CFLAGS_DEBUG += -O2
+    DEFINES *= WINDOWS_SYS
+    PRE_TARGETDEPS += C:\Development\Rs\v0.5-new_cache_system\libretroshare\libretroshare-build-desktop\lib\libretroshare.a
+    LIBS += C:\Development\Rs\v0.5-new_cache_system\libretroshare\libretroshare-build-desktop\lib\libretroshare.a
+    LIBS += -L"../lib"
+    LIBS += -lssl -lcrypto -lgpgme -lpthreadGC2d -lminiupnpc -lz
+# added after bitdht
+#    LIBS += -lws2_32
+        LIBS += -luuid -lole32 -liphlpapi -lcrypt32-cygwin -lgdi32
+        LIBS += -lole32 -lwinmm
+
+        # export symbols for the plugins
+        #LIBS += -Wl,--export-all-symbols,--out-implib,lib/libretroshare-gui.a
+
+    GPG_ERROR_DIR = ../../../../libgpg-error-1.7
+    GPGME_DIR  = ../../../../gpgme-1.1.8
+    GPG_ERROR_DIR = ../../../../lib/libgpg-error-1.7
+    GPGME_DIR  = ../../../../lib/gpgme-1.1.8
+    INCLUDEPATH += . $${GPGME_DIR}/src $${GPG_ERROR_DIR}/src ../../Libraries/sqlite/sqlite-autoconf-3070900
+    LIBS += C:\Development\Libraries\sqlite\sqlite-autoconf-3070900\.libs\libsqlite3.a
+}
+
+win32 {
+# must be added after bitdht
+    LIBS += -lws2_32
+}
+
+SOURCES += \
+    support.cc \
+    #rsnxsitems_test.cc
+    rsdataservice_test.cc \
+    data_support.cc
+    #rsnxsservice_test.cc \
+    #nxstesthub.cc
+    #rsgxsdata_test.cc
+
+HEADERS += support.h \
+    #rsnxsitems_test.h
+    rsdataservice_test.h \
+    data_support.h
+    #rsnxsservice_test.h \
+    #nxstesthub.h
+
+INCLUDEPATH += C:\Development\Rs\v0.5-new_cache_system\libretroshare\src

libretroshare/src/tests/gxs/rsdataservice_test.cc

@@ -0,0 +1,351 @@
+
+#include "support.h"
+#include "data_support.h"
+#include "rsdataservice_test.h"
+#include "gxs/rsdataservice.h"
+
+#define DATA_BASE_NAME "msg_grp_Store"
+
+INITTEST();
+RsGeneralDataService* dStore = NULL;
+
+void setUp();
+void tearDown();
+
+int main()
+{
+
+    std::cerr << "RsDataService Tests" << std::endl;
+
+    test_groupStoreAndRetrieve(); REPORT("test_groupStoreAndRetrieve");
+    test_messageStoresAndRetrieve(); REPORT("test_messageStoresAndRetrieve");
+
+    FINALREPORT("RsDataService Tests");
+
+    return TESTRESULT();
+
+}
+
+
+
+/*!
+ * All memory is disposed off, good for looking
+ * for memory leaks
+ */
+void test_groupStoreAndRetrieve(){
+
+    setUp();
+
+    int nGrp = rand()%32;
+    std::map<RsNxsGrp*, RsGxsGrpMetaData*> grps;
+    RsNxsGrp* grp;
+    RsGxsGrpMetaData* grpMeta;
+    for(int i = 0; i < nGrp; i++){
+        std::pair<RsNxsGrp*, RsGxsGrpMetaData*> p;
+       grp = new RsNxsGrp(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+       grpMeta = new RsGxsGrpMetaData();
+       p.first = grp;
+       p.second = grpMeta;
+       init_item(*grp);
+       init_item(grpMeta);
+       grpMeta->mGroupId = grp->grpId;
+       grps.insert(p);
+
+       grpMeta = NULL;
+       grp = NULL;
+   }
+
+    dStore->storeGroup(grps);
+
+    std::map<std::string, RsNxsGrp*> gR;
+    std::map<std::string, RsGxsGrpMetaData*> grpMetaR;
+    dStore->retrieveNxsGrps(gR, false);
+    dStore->retrieveGxsGrpMetaData(grpMetaR);
+
+    std::map<RsNxsGrp*, RsGxsGrpMetaData*>::iterator mit = grps.begin();
+
+    bool grpMatch = true, grpMetaMatch = true;
+
+    for(; mit != grps.end(); mit++)
+    {
+        const std::string grpId = mit->first->grpId;
+
+        // check if it exists
+        if(gR.find(grpId) == gR.end()) {
+            grpMatch = false;
+            break;
+        }
+
+        RsNxsGrp *l = mit->first,
+        *r = gR[grpId];
+
+        // assign transaction number
+        // to right to as tn is not stored
+        // in db
+        r->transactionNumber = l->transactionNumber;
+
+        // then do a comparison
+        if(!( *l == *r)) {
+            grpMatch = false;
+            break;
+        }
+
+        // now do a comparison of grp meta types
+
+        if(grpMetaR.find(grpId) == grpMetaR.end())
+        {
+            grpMetaMatch = false;
+            break;
+        }
+
+        RsGxsGrpMetaData *l_Meta = mit->second,
+        *r_Meta = grpMetaR[grpId];
+
+        if(!(*l_Meta == *r_Meta))
+        {
+            grpMetaMatch = false;
+            break;
+        }
+
+        /* release resources */
+        delete l_Meta;
+        delete r_Meta;
+        delete l;
+        delete r;
+
+        remove(grpId.c_str());
+    }
+
+    grpMetaR.clear();
+
+    CHECK(grpMatch);
+    tearDown();
+}
+
+/*!
+ * Test for both selective and
+ * bulk msg retrieval
+ */
+void test_messageStoresAndRetrieve()
+{
+    setUp();
+
+    // first create a grpId
+    std::string grpId0, grpId1;
+
+    randString(SHORT_STR, grpId0);
+    randString(SHORT_STR, grpId1);
+    std::vector<std::string> grpV; // stores grpIds of all msgs stored and retrieved
+    grpV.push_back(grpId0);
+    grpV.push_back(grpId1);
+
+    std::map<RsNxsMsg*, RsGxsMsgMetaData*> msgs;
+    RsNxsMsg* msg = NULL;
+    RsGxsMsgMetaData* msgMeta = NULL;
+    int nMsgs = rand()%120;
+    GxsMsgReq req;
+
+    std::map<std::string, RsNxsMsg*> VergrpId0, VergrpId1;
+    std::map<std::string, RsGxsMsgMetaData*> VerMetagrpId0, VerMetagrpId1;
+
+    for(int i=0; i<nMsgs; i++)
+    {
+        msg = new RsNxsMsg(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+        msgMeta = new RsGxsMsgMetaData();
+        init_item(*msg);
+        init_item(msgMeta);
+        std::pair<RsNxsMsg*, RsGxsMsgMetaData*> p(msg, msgMeta);
+        int chosen = 0;
+        if(rand()%50 > 24){
+            chosen = 1;
+
+        }
+
+        const std::string& grpId = grpV[chosen];
+
+        if(chosen)
+            req[grpId].insert(msg->msgId);
+
+        msgMeta->mMsgId = msg->msgId;
+        msgMeta->mGroupId = msg->grpId = grpId;
+
+        // store msgs in map to use for verification
+        std::pair<std::string, RsNxsMsg*> vP(msg->msgId, msg);
+        std::pair<std::string, RsGxsMsgMetaData*> vPmeta(msg->msgId, msgMeta);
+
+        if(!chosen)
+        {
+            VergrpId0.insert(vP);
+            VerMetagrpId0.insert(vPmeta);
+        }
+        else
+        {
+            VergrpId1.insert(vP);
+            VerMetagrpId0.insert(vPmeta);
+        }
+        msg = NULL;
+        msgMeta = NULL;
+
+        msgs.insert(p);
+    }
+
+    req[grpV[0]] = std::set<std::string>(); // assign empty list for other
+
+    dStore->storeMessage(msgs);
+
+    // now retrieve msgs for comparison
+    // first selective retrieval
+
+    GxsMsgResult msgResult;
+    GxsMsgMetaResult msgMetaResult;
+    dStore->retrieveNxsMsgs(req, msgResult, false);
+    dStore->retrieveGxsMsgMetaData(grpV, msgMetaResult);
+
+    // now look at result for grpId 1
+    std::vector<RsNxsMsg*>& result0 = msgResult[grpId0];
+    std::vector<RsNxsMsg*>& result1 = msgResult[grpId1];
+    std::vector<RsGxsMsgMetaData*>& resultMeta0 = msgMetaResult[grpId0];
+    std::vector<RsGxsMsgMetaData*>& resultMeta1 = msgMetaResult[grpId1];
+
+
+
+    bool msgGrpId0_Match = true, msgGrpId1_Match = true;
+    bool msgMetaGrpId0_Match = true, msgMetaGrpId1_Match = true;
+
+    // MSG test, selective retrieval
+    for(std::vector<RsNxsMsg*>::size_type i = 0; i < result0.size(); i++)
+    {
+        RsNxsMsg* l = result0[i] ;
+
+        if(VergrpId0.find(l->msgId) == VergrpId0.end())
+        {
+            msgGrpId0_Match = false;
+            break;
+        }
+
+        RsNxsMsg* r = VergrpId0[l->msgId];
+        r->transactionNumber = l->transactionNumber;
+
+        if(!(*l == *r))
+        {
+            msgGrpId0_Match = false;
+            break;
+        }
+    }
+
+    CHECK(msgGrpId0_Match);
+
+    // META test
+    for(std::vector<RsGxsMsgMetaData*>::size_type i = 0; i < resultMeta0.size(); i++)
+    {
+        RsGxsMsgMetaData* l = resultMeta0[i] ;
+
+        if(VerMetagrpId0.find(l->mMsgId) == VerMetagrpId0.end())
+        {
+            msgMetaGrpId0_Match = false;
+            break;
+        }
+
+        RsGxsMsgMetaData* r = VerMetagrpId0[l->mMsgId];
+
+        if(!(*l == *r))
+        {
+            msgMetaGrpId0_Match = false;
+            break;
+        }
+    }
+
+    CHECK(msgMetaGrpId0_Match);
+
+    // MSG test, bulk retrieval
+    for(std::vector<RsNxsMsg*>::size_type i = 0; i < result1.size(); i++)
+    {
+        RsNxsMsg* l = result1[i] ;
+
+        if(VergrpId1.find(l->msgId) == VergrpId1.end())
+        {
+            msgGrpId1_Match = false;
+            break;
+        }
+
+        RsNxsMsg* r = VergrpId1[l->msgId];
+
+        r->transactionNumber = l->transactionNumber;
+
+        if(!(*l == *r))
+        {
+            msgGrpId1_Match = false;
+            break;
+        }
+    }
+
+    CHECK(msgGrpId1_Match);
+
+    //dStore->retrieveGxsMsgMetaData();
+    std::string msgFile = grpId0 + "-msgs";
+    remove(msgFile.c_str());
+    msgFile = grpId1 + "-msgs";
+    remove(msgFile.c_str());
+    tearDown();
+}
+
+
+
+void setUp(){
+    dStore = new RsDataService(".", DATA_BASE_NAME, RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+void tearDown(){
+
+    dStore->resetDataStore(); // reset to clean up store files except db
+    delete dStore;
+    dStore = NULL;
+    int rc = remove(DATA_BASE_NAME);
+
+    if(rc == 0){
+        std::cerr << "Successful tear down" << std::endl;
+    }
+    else{
+        std::cerr << "Tear down failed" << std::endl;
+        perror("Error: ");
+    }
+
+}
+
+
+
+bool operator ==(const RsGxsGrpMetaData& l, const RsGxsGrpMetaData& r)
+{
+    if(!(l.adminSign == r.adminSign)) return false;
+    if(!(l.idSign == r.idSign)) return false;
+    if(!(l.keys == r.keys)) return false;
+    if(l.mGroupFlags != r.mGroupFlags) return false;
+    if(l.mPublishTs != r.mPublishTs) return false;
+    if(l.mAuthorId != r.mAuthorId) return false;
+    if(l.mGroupName != r.mGroupName) return false;
+    if(l.mGroupId != r.mGroupId) return false;
+    if(l.mGroupStatus != r.mGroupStatus) return false;
+    if(l.mPop != r.mPop) return false;
+    if(l.mMsgCount != r.mMsgCount) return false;
+    if(l.mSubscribeFlags != r.mSubscribeFlags) return false;
+
+    return true;
+}
+
+bool operator ==(const RsGxsMsgMetaData& l, const RsGxsMsgMetaData& r)
+{
+
+    if(!(l.idSign == r.idSign)) return false;
+    if(!(l.pubSign == r.pubSign)) return false;
+    if(l.mGroupId != r.mGroupId) return false;
+    if(l.mAuthorId != r.mAuthorId) return false;
+    if(l.mParentId != r.mParentId) return false;
+    if(l.mOrigMsgId != r.mOrigMsgId) return false;
+    if(l.mThreadId != r.mThreadId) return false;
+    if(l.mMsgId != r.mMsgId) return false;
+    if(l.mMsgName != r.mMsgName) return false;
+    if(l.mPublishTs != r.mPublishTs) return false;
+    if(l.mMsgFlags != r.mMsgFlags) return false;
+
+    return true;
+}

libretroshare/src/tests/gxs/rsdataservice_test.h

@@ -0,0 +1,33 @@
+#ifndef RSDATASERVICE_TEST_H
+#define RSDATASERVICE_TEST_H
+
+#include "util/rsthreads.h"
+#include "serialiser/rsnxsitems.h"
+#include "gxs/rsgds.h"
+
+void test_messageStoresAndRetrieve();
+
+void test_groupStoreAndRetrieve();
+
+void test_storeAndDeleteGroup();
+void test_storeAndDeleteMessage();
+
+void test_searchMsg();
+void test_searchGrp();
+
+bool operator ==(const RsGxsGrpMetaData& l, const RsGxsGrpMetaData& r);
+bool operator ==(const RsGxsMsgMetaData& l, const RsGxsMsgMetaData& r);
+
+void test_multiThreaded();
+
+class DataReadWrite : RsThread
+{
+
+
+
+};
+
+void test_cacheSize();
+
+
+#endif // RSDATASERVICE_TEST_H

libretroshare/src/tests/gxs/rsgxsdata_test.cc

@@ -0,0 +1,80 @@
+
+#include "support.h"
+#include "data_support.h"
+#include "gxs/rsgxsdata.h"
+#include "util/utest.h"
+
+INITTEST();
+
+bool operator ==(const RsGxsGrpMetaData& l, const RsGxsGrpMetaData& r);
+bool operator ==(const RsGxsMsgMetaData& l, const RsGxsMsgMetaData& r);
+
+int main()
+{
+
+    RsGxsGrpMetaData grpMeta1, grpMeta2;
+    RsGxsMsgMetaData msgMeta1, msgMeta2;
+
+    grpMeta1.clear();
+    init_item(&grpMeta1);
+
+    msgMeta1.clear();
+    init_item(&msgMeta1);
+
+    uint32_t pktsize = grpMeta1.serial_size();
+    char grp_data[pktsize];
+
+    bool ok = true;
+
+    ok &= grpMeta1.serialise(grp_data, pktsize);
+    grpMeta2.clear();
+    ok &= grpMeta2.deserialise(grp_data, pktsize);
+
+    CHECK(grpMeta1 == grpMeta2);
+
+    pktsize = msgMeta1.serial_size();
+    char msg_data[pktsize];
+
+    ok &= msgMeta1.serialise(msg_data, &pktsize);
+    msgMeta2.clear();
+    ok &= msgMeta2.deserialise(msg_data, &pktsize);
+
+    CHECK(msgMeta1 == msgMeta2);
+
+    FINALREPORT("GxsMeta Data Test");
+
+    return TESTRESULT();
+}
+
+
+bool operator ==(const RsGxsGrpMetaData& l, const RsGxsGrpMetaData& r)
+{
+    if(!(l.adminSign == r.adminSign)) return false;
+    if(!(l.idSign == r.idSign)) return false;
+    if(!(l.keys == r.keys)) return false;
+    if(l.mGroupFlags != r.mGroupFlags) return false;
+    if(l.mPublishTs != r.mPublishTs) return false;
+    if(l.mAuthorId != r.mAuthorId) return false;
+    if(l.mGroupName != r.mGroupName) return false;
+    if(l.mGroupId != r.mGroupId) return false;
+
+    return true;
+}
+
+bool operator ==(const RsGxsMsgMetaData& l, const RsGxsMsgMetaData& r)
+{
+
+    if(!(l.idSign == r.idSign)) return false;
+    if(!(l.pubSign == r.pubSign)) return false;
+    if(l.mGroupId != r.mGroupId) return false;
+    if(l.mAuthorId != r.mAuthorId) return false;
+    if(l.mParentId != r.mParentId) return false;
+    if(l.mOrigMsgId != r.mOrigMsgId) return false;
+    if(l.mThreadId != r.mThreadId) return false;
+    if(l.mMsgId != r.mMsgId) return false;
+    if(l.mMsgName != r.mMsgName) return false;
+    if(l.mPublishTs != r.mPublishTs) return false;
+    if(l.mMsgFlags != r.mMsgFlags) return false;
+
+    return true;
+}

libretroshare/src/tests/gxs/rsgxsnetservice_test.cc

@@ -0,0 +1,36 @@
+/*
+ * rsgxsnetservice_test.cc
+ *
+ *  Created on: 11 Jul 2012
+ *      Author: crispy
+ */
+
+#include "util/utest.h"
+#include "nxstesthub.h"
+#include "nxstestscenario.h"
+
+INITTEST();
+
+
+int main()
+{
+
+	// first setup
+	NxsMessageTest msgTest(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+	NxsTestHub hub(&msgTest);
+
+	// now get things started
+	createThread(hub);
+
+	// put this thread to sleep for 10 secs
+	sleep(10);
+
+	hub.join();
+	CHECK(hub.testsPassed());
+
+	hub.cleanUp();
+
+    FINALREPORT("RsGxsNetService Tests");
+
+    return TESTRESULT();
+}

libretroshare/src/tests/gxs/support.cc

@@ -0,0 +1,303 @@
+/*
+ * libretroshare/src/serialiser: t_support.h.cc
+ *
+ * RetroShare Serialiser tests.
+ *
+ * Copyright 2007-2008 by Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <stdlib.h>
+
+#include "support.h"
+#include "serialiser/rstlvbase.h"
+
+void randString(const uint32_t length, std::string& outStr)
+{
+	char alpha = 'a';
+	char* stringData = NULL;
+
+	stringData = new char[length];
+
+	for(uint32_t i=0; i != length; i++)
+		stringData[i] = alpha + (rand() % 26);
+
+	outStr.assign(stringData, length);
+	delete[] stringData;
+
+	return;
+}
+
+void randString(const uint32_t length, std::wstring& outStr)
+{
+	wchar_t alpha = L'a';
+	wchar_t* stringData = NULL;
+
+	stringData = new wchar_t[length];
+
+	for(uint32_t i=0; i != length; i++)
+		stringData[i] = (alpha + (rand() % 26));
+
+	outStr.assign(stringData, length);
+	delete[] stringData;
+
+	return;
+}
+
+void init_item(RsTlvSecurityKeySet& ks)
+{
+    int n = rand()%24;
+    randString(SHORT_STR, ks.groupId);
+    for(int i=1; i<n; i++)
+    {
+        std::string a_str;
+        randString(SHORT_STR, a_str);
+
+        RsTlvSecurityKey& a_key = ks.keys[a_str];
+        init_item(a_key);
+        a_key.keyId = a_str;
+    }
+}
+
+bool operator==(const RsTlvSecurityKeySet& l, const RsTlvSecurityKeySet& r)
+{
+
+    if(l.groupId != r.groupId) return false;
+
+    std::map<std::string, RsTlvSecurityKey>::const_iterator l_cit = l.keys.begin(),
+    r_cit = r.keys.begin();
+
+    for(; l_cit != l.keys.end(); l_cit++, r_cit++){
+        if(l_cit->first != r_cit->first) return false;
+        if(!(l_cit->second == r_cit->second)) return false;
+    }
+
+    return true;
+}
+
+bool operator==(const RsTlvSecurityKey& sk1, const RsTlvSecurityKey& sk2)
+{
+
+	if(sk1.startTS != sk2.startTS) return false;
+	if(sk1.endTS != sk2.endTS) return false;
+	if(sk1.keyFlags != sk2.keyFlags) return false;
+	if(sk1.keyId != sk2.keyId) return false;
+	if(!(sk1.keyData == sk1.keyData)) return false;
+
+	return true;
+}
+
+bool operator==(const RsTlvKeySignature& ks1, const RsTlvKeySignature& ks2)
+{
+
+	if(ks1.keyId != ks2.keyId) return false;
+	if(!(ks1.signData == ks2.signData)) return false;
+
+	return true;
+}
+
+bool operator==(const RsTlvPeerIdSet& pids1, const RsTlvPeerIdSet& pids2)
+{
+	std::list<std::string>::const_iterator it1 = pids1.ids.begin(),
+			it2 = pids2.ids.begin();
+
+
+	for(; ((it1 != pids1.ids.end()) && (it2 != pids2.ids.end())); it1++, it2++)
+	{
+		if(*it1 != *it2) return false;
+	}
+
+	return true;
+
+}
+
+
+void init_item(RsTlvImage& im)
+{
+	std::string imageData;
+	randString(LARGE_STR, imageData);
+	im.binData.setBinData(imageData.c_str(), imageData.size());
+	im.image_type = RSTLV_IMAGE_TYPE_PNG;
+
+	return;
+}
+
+bool operator==(const RsTlvBinaryData& bd1, const RsTlvBinaryData& bd2)
+{
+	if(bd1.tlvtype != bd2.tlvtype) return false;
+	if(bd1.bin_len != bd2.bin_len) return false;
+
+	unsigned char *bin1 = (unsigned char*)(bd1.bin_data),
+			*bin2 = (unsigned char*)(bd2.bin_data);
+
+	for(uint32_t i=0; i < bd1.bin_len; bin1++, bin2++, i++)
+	{
+		if(*bin1 != *bin2)
+			return false;
+	}
+
+	return true;
+}
+
+
+void init_item(RsTlvSecurityKey& sk)
+{
+	int randnum = rand()%313131;
+
+	sk.endTS = randnum;
+	sk.keyFlags = randnum;
+	sk.startTS = randnum;
+	randString(SHORT_STR, sk.keyId);
+
+	std::string randomStr;
+	randString(LARGE_STR, randomStr);
+
+	sk.keyData.setBinData(randomStr.c_str(), randomStr.size());
+
+	return;
+}
+
+void init_item(RsTlvKeySignature& ks)
+{
+	randString(SHORT_STR, ks.keyId);
+
+	std::string signData;
+	randString(LARGE_STR, signData);
+
+	ks.signData.setBinData(signData.c_str(), signData.size());
+
+	return;
+}
+
+
+bool operator==(const RsTlvImage& img1, const RsTlvImage& img2)
+{
+	if(img1.image_type != img2.image_type) return false;
+	if(!(img1.binData == img2.binData)) return false;
+
+	return true;
+
+}
+
+/** channels, forums and blogs **/
+
+void init_item(RsTlvHashSet& hs)
+{
+	std::string hash;
+
+	for(int i=0; i < 10; i++)
+	{
+		randString(SHORT_STR, hash);
+		hs.ids.push_back(hash);
+	}
+
+	hs.mType = TLV_TYPE_HASHSET;
+	return;
+}
+
+void init_item(RsTlvPeerIdSet& ps)
+{
+	std::string peer;
+
+	for(int i=0; i < 10; i++)
+	{
+		randString(SHORT_STR, peer);
+		ps.ids.push_back(peer);
+	}
+
+	ps.mType = TLV_TYPE_PEERSET;
+	return;
+}
+
+bool operator==(const RsTlvHashSet& hs1,const RsTlvHashSet& hs2)
+{
+	if(hs1.mType != hs2.mType) return false;
+
+	std::list<std::string>::const_iterator it1 = hs1.ids.begin(),
+			it2 = hs2.ids.begin();
+
+	for(; ((it1 != hs1.ids.end()) && (it2 != hs2.ids.end())); it1++, it2++)
+	{
+		if(*it1 != *it2) return false;
+	}
+
+	return true;
+}
+
+void init_item(RsTlvFileItem& fi)
+{
+	fi.age = rand()%200;
+	fi.filesize = rand()%34030313;
+	randString(SHORT_STR, fi.hash);
+	randString(SHORT_STR, fi.name);
+	randString(SHORT_STR, fi.path);
+	fi.piecesize = rand()%232;
+	fi.pop = rand()%2354;
+	init_item(fi.hashset);
+
+	return;
+}
+
+void init_item(RsTlvBinaryData& bd){
+    bd.TlvClear();
+    std::string data;
+    randString(LARGE_STR, data);
+    bd.setBinData(data.data(), data.length());
+}
+
+void init_item(RsTlvFileSet& fSet){
+
+	randString(LARGE_STR, fSet.comment);
+	randString(SHORT_STR, fSet.title);
+	RsTlvFileItem fi1, fi2;
+	init_item(fi1);
+	init_item(fi2);
+	fSet.items.push_back(fi1);
+	fSet.items.push_back(fi2);
+
+	return;
+}
+
+bool operator==(const RsTlvFileSet& fs1,const  RsTlvFileSet& fs2)
+{
+	if(fs1.comment != fs2.comment) return false;
+	if(fs1.title != fs2.title) return false;
+
+	std::list<RsTlvFileItem>::const_iterator it1 = fs1.items.begin(),
+			it2 = fs2.items.begin();
+
+	for(;  ((it1 != fs1.items.end()) && (it2 != fs2.items.end())); it1++, it2++)
+		if(!(*it1 == *it2)) return false;
+
+	return true;
+}
+
+bool operator==(const RsTlvFileItem& fi1,const RsTlvFileItem& fi2)
+{
+	if(fi1.age != fi2.age) return false;
+	if(fi1.filesize != fi2.filesize) return false;
+	if(fi1.hash != fi2.hash) return false;
+	if(!(fi1.hashset == fi2.hashset)) return false;
+	if(fi1.name != fi2.name) return false;
+	if(fi1.path != fi2.path) return false;
+	if(fi1.piecesize != fi2.piecesize) return false;
+	if(fi1.pop != fi2.pop) return false;
+
+	return true;
+}

libretroshare/src/tests/gxs/support.h

@@ -0,0 +1,224 @@
+#ifndef SUPPORT_H_
+#define SUPPORT_H_
+
+/*
+ * libretroshare/src/tests/serialiser:
+ *
+ * RetroShare Serialiser tests.
+ *
+ * Copyright 2007-2008 by Christopher Evi-Parker, Cyril Soler
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <string>
+#include <stdint.h>
+#include <iostream>
+
+#include "util/utest.h"
+#include "serialiser/rsserial.h"
+#include "serialiser/rstlvutil.h"
+#include "serialiser/rstlvkeys.h"
+#include "serialiser/rstlvtypes.h"
+
+
+
+/**
+ * This contains functions that may be useful for testing throughout the
+ * retroshare's serialiser
+ * package, if you find a function you keep using everywhere, might be a good idea to add it here
+ */
+
+
+#define SHORT_STR 100
+#define LARGE_STR 1000
+
+void randString(const uint32_t, std::string&);
+void randString(const uint32_t, std::wstring&);
+
+
+/* for testing compound tlv items */
+
+void init_item(RsTlvSecurityKey&);
+void init_item(RsTlvKeySignature&);
+void init_item(RsTlvBinaryData&);
+void init_item(RsTlvFileItem&);
+void init_item(RsTlvFileSet&);
+void init_item(RsTlvHashSet&);
+void init_item(RsTlvPeerIdSet&);
+void init_item(RsTlvImage&);
+void init_item(RsTlvPeerIdSet&);
+void init_item(RsTlvSecurityKeySet& );
+
+bool operator==(const RsTlvSecurityKey&, const RsTlvSecurityKey& );
+bool operator==(const RsTlvKeySignature&, const RsTlvKeySignature& );
+bool operator==(const RsTlvBinaryData&, const RsTlvBinaryData&);
+bool operator==(const RsTlvFileItem&, const RsTlvFileItem&);
+bool operator==(const RsTlvFileSet&, const RsTlvFileSet& );
+bool operator==(const RsTlvHashSet&, const RsTlvHashSet&);
+bool operator==(const RsTlvImage&, const RsTlvImage& );
+bool operator==(const RsTlvPeerIdSet& , const RsTlvPeerIdSet& );
+bool operator==(const RsTlvSecurityKeySet& , const RsTlvSecurityKeySet& );
+
+
+
+/*!
+ * This templated test function which allows you to test
+ * retroshare serialiser items (except compound tlv items)
+ * you the function must implement a function
+ *
+ * 'RsSerialType* init_item(YourRsItem& rs_item)'
+ * which returns valid serialiser that
+ * can serialiser rs_item. You also need to implement an operator
+ *
+ * Also need to implement a function
+ * 'bool operator =(YourRsItem& rs_itemL, YourRsItem& rs_temR)'
+ * which allows this  function to test for equality between both parameters.
+ * rs_temR is the result of deserialising the left operand (1st parameter).
+ * not YourRsItem in specifier in about functions should be a derived type from RsItem
+ *
+ * @param T the item you want to test
+ */
+
+template<class T> int test_RsItem()
+{
+	/* make a serialisable RsTurtleItem */
+
+	RsSerialiser srl;
+
+	/* initialise */
+	T rsfi ;
+	RsSerialType *rsfis = init_item(rsfi) ;
+
+	/* attempt to serialise it before we add it to the serialiser */
+
+	CHECK(0 == srl.size(&rsfi));
+
+	static const uint32_t MAX_BUFSIZE = 22000 ;
+
+	char *buffer = new char[MAX_BUFSIZE];
+	uint32_t sersize = MAX_BUFSIZE;
+
+	CHECK(false == srl.serialise(&rsfi, (void *) buffer, &sersize));
+
+	/* now add to serialiser */
+
+	srl.addSerialType(rsfis);
+
+	uint32_t size = srl.size(&rsfi);
+	bool done = srl.serialise(&rsfi, (void *) buffer, &sersize);
+
+	std::cerr << "test_Item() size: " << size << std::endl;
+	std::cerr << "test_Item() done: " << done << std::endl;
+	std::cerr << "test_Item() sersize: " << sersize << std::endl;
+
+	std::cerr << "test_Item() serialised:" << std::endl;
+	//displayRawPacket(std::cerr, (void *) buffer, sersize);
+
+	CHECK(done == true);
+
+	uint32_t sersize2 = sersize;
+	RsItem *output = srl.deserialise((void *) buffer, &sersize2);
+
+	CHECK(output != NULL);
+	CHECK(sersize2 == sersize);
+
+	T *outfi = dynamic_cast<T *>(output);
+
+	CHECK(outfi != NULL);
+
+	if (outfi)
+		CHECK(*outfi == rsfi) ;
+
+	sersize2 = MAX_BUFSIZE;
+	bool done2 = srl.serialise(outfi, (void *) &(buffer[16*8]), &sersize2);
+
+	CHECK(done2) ;
+	CHECK(sersize2 == sersize);
+
+//	displayRawPacket(std::cerr, (void *) buffer, 16 * 8 + sersize2);
+
+	delete[] buffer ;
+	//delete rsfis;
+
+	return 1;
+}
+
+template<class T> int test_RsItem(uint16_t servtype)
+{
+        /* make a serialisable RsTurtleItem */
+
+        RsSerialiser srl;
+
+        /* initialise */
+        T rsfi(servtype) ;
+        RsSerialType *rsfis = init_item(rsfi) ; // deleted on destruction of srl
+
+        /* attempt to serialise it before we add it to the serialiser */
+
+        CHECK(0 == srl.size(&rsfi));
+
+        static const uint32_t MAX_BUFSIZE = 22000 ;
+
+        char *buffer = new char[MAX_BUFSIZE];
+        uint32_t sersize = MAX_BUFSIZE;
+
+        CHECK(false == srl.serialise(&rsfi, (void *) buffer, &sersize));
+
+        /* now add to serialiser */
+
+        srl.addSerialType(rsfis);
+
+        uint32_t size = srl.size(&rsfi);
+        bool done = srl.serialise(&rsfi, (void *) buffer, &sersize);
+
+        std::cerr << "test_Item() size: " << size << std::endl;
+        std::cerr << "test_Item() done: " << done << std::endl;
+        std::cerr << "test_Item() sersize: " << sersize << std::endl;
+
+        std::cerr << "test_Item() serialised:" << std::endl;
+        //displayRawPacket(std::cerr, (void *) buffer, sersize);
+
+        CHECK(done == true);
+
+        uint32_t sersize2 = sersize;
+        RsItem *output = srl.deserialise((void *) buffer, &sersize2);
+
+        CHECK(output != NULL);
+        CHECK(sersize2 == sersize);
+
+        T *outfi = dynamic_cast<T *>(output);
+
+        CHECK(outfi != NULL);
+
+        if (outfi)
+                CHECK(*outfi == rsfi) ;
+
+        sersize2 = MAX_BUFSIZE;
+        bool done2 = srl.serialise(outfi, (void *) &(buffer[16*8]), &sersize2);
+
+        CHECK(done2) ;
+        CHECK(sersize2 == sersize);
+
+//	displayRawPacket(std::cerr, (void *) buffer, 16 * 8 + sersize2);
+
+        delete[] buffer ;
+
+        return 1;
+}
+#endif /* SUPPORT_H_ */

libretroshare/src/tests/serialiser/rsnxsitems_test.cc

@@ -0,0 +1,187 @@
+
+#include "support.h"
+
+#include "rsnxsitems_test.h"
+
+INITTEST();
+
+#define NUM_BIN_OBJECTS 5
+#define NUM_SYNC_MSGS 8
+#define NUM_SYNC_GRPS 5
+
+RsSerialType* init_item(RsNxsGrp& nxg)
+{
+    nxg.clear();
+
+    randString(SHORT_STR, nxg.grpId);
+    nxg.transactionNumber = rand()%23;
+    init_item(nxg.grp);
+    init_item(nxg.meta);
+
+    return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+
+RsSerialType* init_item(RsNxsMsg& nxm)
+{
+    nxm.clear();
+
+    randString(SHORT_STR, nxm.msgId);
+    randString(SHORT_STR, nxm.grpId);
+    init_item(nxm.msg);
+    init_item(nxm.meta);
+    nxm.transactionNumber = rand()%23;
+
+    return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+RsSerialType* init_item(RsNxsSyncGrp& rsg)
+{
+    rsg.clear();
+    rsg.flag = RsNxsSyncGrp::FLAG_USE_SYNC_HASH;
+    rsg.syncAge = rand()%2423;
+    randString(3124,rsg.syncHash);
+
+    return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+RsSerialType* init_item(RsNxsSyncMsg& rsgm)
+{
+    rsgm.clear();
+
+    rsgm.flag = RsNxsSyncMsg::FLAG_USE_SYNC_HASH;
+    rsgm.syncAge = rand()%24232;
+    rsgm.transactionNumber = rand()%23;
+    randString(SHORT_STR, rsgm.grpId);
+    randString(SHORT_STR, rsgm.syncHash);
+
+    return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+RsSerialType* init_item(RsNxsSyncGrpItem& rsgl)
+{
+    rsgl.clear();
+
+    rsgl.flag = RsNxsSyncGrpItem::FLAG_RESPONSE;
+    rsgl.transactionNumber = rand()%23;
+    rsgl.publishTs = rand()%23;
+    randString(SHORT_STR, rsgl.grpId);
+
+    return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+RsSerialType* init_item(RsNxsSyncMsgItem& rsgml)
+{
+    rsgml.clear();
+
+    rsgml.flag = RsNxsSyncGrpItem::FLAG_RESPONSE;
+    rsgml.transactionNumber = rand()%23;
+    randString(SHORT_STR, rsgml.grpId);
+    randString(SHORT_STR, rsgml.msgId);
+
+    return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+RsSerialType* init_item(RsNxsTransac& rstx){
+
+    rstx.clear();
+
+    rstx.timestamp = rand()%14141;
+    rstx.transactFlag = rand()%2424;
+    rstx.nItems = rand()%33132;
+    rstx.transactionNumber = rand()%242112;
+
+    return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM);
+}
+
+bool operator==(const RsNxsGrp& l, const RsNxsGrp& r){
+
+    if(l.grpId != r.grpId) return false;
+    if(!(l.grp == r.grp) ) return false;
+    if(!(l.meta == r.meta) ) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+bool operator==(const RsNxsMsg& l, const RsNxsMsg& r){
+
+    if(l.msgId != r.msgId) return false;
+    if(l.grpId != r.grpId) return false;
+    if(! (l.msg == r.msg) ) return false;
+    if(! (l.meta == r.meta) ) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+bool operator==(const RsNxsSyncGrp& l, const RsNxsSyncGrp& r)
+{
+
+    if(l.syncHash != r.syncHash) return false;
+    if(l.flag != r.flag) return false;
+    if(l.syncAge != r.syncAge) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+bool operator==(const RsNxsSyncMsg& l, const RsNxsSyncMsg& r)
+{
+
+    if(l.flag != r.flag) return false;
+    if(l.syncAge != r.syncAge) return false;
+    if(l.syncHash != r.syncHash) return false;
+    if(l.grpId != r.grpId) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+bool operator==(const RsNxsSyncGrpItem& l, const RsNxsSyncGrpItem& r)
+{
+    if(l.flag != r.flag) return false;
+    if(l.publishTs != r.publishTs) return false;
+    if(l.grpId != r.grpId) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+bool operator==(const RsNxsSyncMsgItem& l, const RsNxsSyncMsgItem& r)
+{
+    if(l.flag != r.flag) return false;
+    if(l.grpId != r.grpId) return false;
+    if(l.msgId != r.msgId) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+
+    return true;
+}
+
+bool operator==(const RsNxsTransac& l, const RsNxsTransac& r){
+
+    if(l.transactFlag != r.transactFlag) return false;
+    if(l.transactionNumber != r.transactionNumber) return false;
+    if(l.timestamp != r.timestamp) return false;
+    if(l.nItems != r.nItems) return false;
+
+
+    return true;
+}
+
+int main()
+{
+    std::cerr << "RsNxsItem Tests" << std::endl;
+
+    test_RsItem<RsNxsGrp>(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsGrpResp");
+    test_RsItem<RsNxsMsg>(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsGrpMsgResp");
+    test_RsItem<RsNxsSyncGrp>(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsNxsSyncGrp");
+    test_RsItem<RsNxsSyncMsg>(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsNxsSyncMsg");
+    test_RsItem<RsNxsSyncGrpItem>(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsNxsSyncGrpItem");
+    test_RsItem<RsNxsSyncMsgItem>(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsNxsSyncMsgItem");
+    test_RsItem<RsNxsTransac>(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsNxsTransac");
+
+    FINALREPORT("RsNxsItem Tests");
+
+    return TESTRESULT();
+}

libretroshare/src/tests/serialiser/rsnxsitems_test.h

@@ -0,0 +1,24 @@
+#ifndef RSNXSITEMS_TEST_H
+#define RSNXSITEMS_TEST_H
+
+#include "serialiser/rsnxsitems.h"
+
+
+RsSerialType* init_item(RsNxsGrp&);
+RsSerialType* init_item(RsNxsMsg&);
+RsSerialType* init_item(RsNxsSyncGrp&);
+RsSerialType* init_item(RsNxsSyncMsg&);
+RsSerialType* init_item(RsNxsSyncGrpItem&);
+RsSerialType* init_item(RsNxsSyncMsgItem&);
+RsSerialType* init_item(RsNxsTransac& );
+
+bool operator==(const RsNxsGrp&, const RsNxsGrp&);
+bool operator==(const RsNxsMsg&, const RsNxsMsg&);
+bool operator==(const RsNxsSyncGrp&, const RsNxsSyncGrp&);
+bool operator==(const RsNxsSyncMsg&, const RsNxsSyncMsg&);
+bool operator==(const RsNxsSyncGrpItem&, const RsNxsSyncGrpItem&);
+bool operator==(const RsNxsSyncMsgItem&, const RsNxsSyncMsgItem&);
+bool operator==(const RsNxsTransac&, const RsNxsTransac& );
+
+
+#endif // RSNXSITEMS_TEST_H

libretroshare/src/tests/serialiser/support.cc

@@ -223,6 +223,13 @@ void init_item(RsTlvFileItem& fi)
 	return;
 }
 
+void init_item(RsTlvBinaryData& bd){
+    bd.TlvClear();
+    std::string data;
+    randString(LARGE_STR, data);
+    bd.setBinData(data.data(), data.length());
+}
+
 void init_item(RsTlvFileSet& fSet){
 
 	randString(LARGE_STR, fSet.comment);

libretroshare/src/tests/serialiser/support.h

@@ -157,5 +157,66 @@ template<class T> int test_RsItem()
 	return 1;
 }
 
+template<class T> int test_RsItem(uint16_t servtype)
+{
+        /* make a serialisable RsTurtleItem */
 
+        RsSerialiser srl;
+
+        /* initialise */
+        T rsfi(servtype) ;
+        RsSerialType *rsfis = init_item(rsfi) ; // deleted on destruction of srl
+
+        /* attempt to serialise it before we add it to the serialiser */
+
+        CHECK(0 == srl.size(&rsfi));
+
+        static const uint32_t MAX_BUFSIZE = 22000 ;
+
+        char *buffer = new char[MAX_BUFSIZE];
+        uint32_t sersize = MAX_BUFSIZE;
+
+        CHECK(false == srl.serialise(&rsfi, (void *) buffer, &sersize));
+
+        /* now add to serialiser */
+
+        srl.addSerialType(rsfis);
+
+        uint32_t size = srl.size(&rsfi);
+        bool done = srl.serialise(&rsfi, (void *) buffer, &sersize);
+
+        std::cerr << "test_Item() size: " << size << std::endl;
+        std::cerr << "test_Item() done: " << done << std::endl;
+        std::cerr << "test_Item() sersize: " << sersize << std::endl;
+
+        std::cerr << "test_Item() serialised:" << std::endl;
+        //displayRawPacket(std::cerr, (void *) buffer, sersize);
+
+        CHECK(done == true);
+
+        uint32_t sersize2 = sersize;
+        RsItem *output = srl.deserialise((void *) buffer, &sersize2);
+
+        CHECK(output != NULL);
+        CHECK(sersize2 == sersize);
+
+        T *outfi = dynamic_cast<T *>(output);
+
+        CHECK(outfi != NULL);
+
+        if (outfi)
+                CHECK(*outfi == rsfi) ;
+
+        sersize2 = MAX_BUFSIZE;
+        bool done2 = srl.serialise(outfi, (void *) &(buffer[16*8]), &sersize2);
+
+        CHECK(done2) ;
+        CHECK(sersize2 == sersize);
+
+//	displayRawPacket(std::cerr, (void *) buffer, 16 * 8 + sersize2);
+
+        delete[] buffer ;
+
+        return 1;
+}
 #endif /* SUPPORT_H_ */

libretroshare/src/tests/util/testcontentvalue.cpp

@@ -0,0 +1,374 @@
+
+#include <memory.h>
+
+#include "retrodb.h"
+#include "utest.h"
+
+bool operator==(const ContentValue& lCV, ContentValue& rCV){
+
+    std::map<std::string, uint8_t> leftKtMap, rightKtMap;
+    lCV.getKeyTypeMap(leftKtMap);
+    rCV.getKeyTypeMap(rightKtMap);
+
+    return (leftKtMap == rightKtMap);
+}
+
+
+/*!
+ * Test content value ability to take a set of data and ability to get it out
+ * For all data types
+ */
+void testEnterAndRetrieve();
+
+/*!
+ * Check to see if copy constructor intialises
+ * 'this' correctly
+ */
+void testCopyConstructor();
+
+/*!
+ * check that key type map returned is consistent
+ * with data contained in ContentValue
+ */
+void testGetKeyTypeMap();
+
+/*!
+ * Test the clearing functionality
+ */
+void testClear();
+
+/*!
+ * check errors are meaningful
+ */
+void testErrors();
+
+/*!
+ * enter the same key twice and ensure previous data gets overwritten
+ *
+ */
+void testSameKey();
+
+
+INITTEST();
+
+int main(){
+
+    testEnterAndRetrieve();
+    testCopyConstructor();
+    testClear();
+    testSameKey();
+    testGetKeyTypeMap();
+    testErrors();
+
+    FINALREPORT("TEST_CONTENT_VALUE");
+}
+
+
+void testSameKey(){
+
+
+    ContentValue cv;
+
+    // test data
+
+    std::string src = "adalkjalfalfalfkal";
+    const char* data = src.data();
+    uint32_t data_len = src.length();
+    std::string data_key = "private_key";
+    cv.put(data_key, data_len, data);
+
+    std::string other_src = "daldko5202402224";
+    data_len = other_src.length();
+    data = other_src.data();
+
+    cv.put(data_key, data_len, data);
+
+    uint32_t val_len;
+    char* val;
+    cv.getAsData(data_key, val_len, val);
+    std::string str_val(val, val_len);
+
+
+    CHECK(str_val == other_src);
+
+    // now check string
+
+    std::string key = "peer";
+    cv.put(key, std::string("sexy_girl"));
+    cv.put(key, std::string("manly man"));
+
+    std::string val_str;
+    cv.getAsString(key, val_str);
+
+    CHECK(val_str == "manly man");
+
+    // and double
+
+    cv.put(key, 4.);
+    cv.put(key, 32.);
+    double val_d;
+    cv.getAsDouble(key, val_d);
+    CHECK(val_d == 32.);
+
+    // and int64
+    int64_t large(20420492040123);
+    cv.put(key, large);
+    cv.put(key, large+34);
+    int64_t val_i64;
+    cv.getAsInt64(key, val_i64);
+
+    CHECK(val_i64 == large+34);
+
+    // and bool
+
+    cv.put(key, false);
+    cv.put(key, true);
+    bool bool_val = false;
+    cv.getAsBool(key, bool_val);
+
+    CHECK(bool_val == true);
+
+    // and int32
+
+    int32_t medium = 20432123;
+    cv.put(key, medium);
+    cv.put(key, medium+34);
+    int32_t val_i32;
+    cv.getAsInt32(key, val_i32);
+
+    CHECK(val_i32 == medium+34);
+
+    REPORT("testSameKey()");
+}
+
+
+void testErrors(){
+
+
+    ContentValue cv;
+    int32_t val_32;
+    int64_t val_64;
+    char* data;
+    uint32_t data_len;
+    std::string val_str;
+    bool val_bool;
+    double val_d;
+    CHECK(!cv.getAsInt32("dda", val_32));
+    CHECK(!cv.getAsInt64("dda", val_64));
+    CHECK(!cv.getAsData("ds", data_len, data));
+    CHECK(!cv.getAsString("d3536356336356356s", val_str));
+    CHECK(!cv.getAsBool("d424s", val_bool));
+    CHECK(!cv.getAsDouble("daads", val_d));
+
+    REPORT("testErrors()");
+}
+
+void testEnterAndRetrieve(){
+
+    // INT 32
+    int32_t value32 = 1, retval32;
+    std::string key = "one";
+
+    ContentValue cv;
+    cv.put(key, value32);
+
+    cv.getAsInt32(key, retval32);
+    CHECK(value32 == retval32);
+
+    // INT 64
+    int64_t value64 = 423425242, retval64;
+    key = "int64";
+    cv.put(key, value64);
+    cv.getAsInt64(key, retval64);
+    CHECK(value64 == retval64);
+
+
+    // Double
+    double dvalue = 3.5, retvaldbl;
+    key = "double";
+    cv.put(key, dvalue);
+    cv.getAsDouble(key, retvaldbl);
+    CHECK(dvalue == retvaldbl );
+
+    // BLOB
+    uint32_t data_len = 45, get_len = 0;
+
+    char* src = new char[data_len];
+    char* get_src = NULL;
+    memset(src, '$', data_len);
+    key = "data";
+    cv.put(key, data_len, src);
+
+     cv.getAsData(key, get_len, get_src);
+
+    bool fine = true;
+
+    CHECK(get_len = data_len);
+
+    if(data_len == get_len){
+
+        for(int i=0; i < data_len; i++){
+
+            if(src[i] != get_src[i])
+                fine &= false;
+
+        }
+    }
+
+    delete[] src;
+    CHECK(fine);
+
+    // STRING
+    std::string strVal = "whoo", getVal("");
+    key = "string";
+    cv.put(key, strVal);
+    cv.getAsString(key, getVal);
+
+    CHECK(getVal == strVal);
+
+
+    // BOOL
+    bool mefalse = false, retvalBool;
+    key = "bool";
+    cv.put(key, mefalse);
+    cv.getAsBool(key, retvalBool);
+    CHECK(mefalse == retvalBool);
+
+    cv.clear();
+
+    REPORT("testEnterAndRetrieve()");
+}
+
+void testGetKeyTypeMap(){
+
+    ContentValue cv;
+    std::string key1="key1", key2="key2", key3="key3", key4="key4";
+    std::string key1_val;
+    int32_t key2_val = 42;
+    double key3_val = 23;
+    int64_t key4_val = 42052094224;
+
+    cv.put(key1, key1_val);
+    cv.put(key2, key2_val);
+    cv.put(key3, key3_val);
+    cv.put(key4, key4_val);
+
+    std::map<std::string, uint8_t> kvMap;
+
+    cv.getKeyTypeMap(kvMap);
+
+    CHECK(kvMap.size() == 4);
+    CHECK(kvMap[key1] == ContentValue::STRING_TYPE);
+    CHECK(kvMap[key2] == ContentValue::INT32_TYPE);
+    CHECK(kvMap[key3] == ContentValue::DOUBLE_TYPE);
+    CHECK(kvMap[key4] == ContentValue::INT64_TYPE);
+
+    REPORT("testGetKeyTypeMap()");
+}
+
+void testCopyConstructor(){
+
+    ContentValue cv1;
+
+    // INT 32
+    int value32 = 1;
+    std::string key = "one";
+
+    cv1.put(key, value32);
+
+    // INT 64
+    int64_t value64 = 423425242;
+    key = "int64";
+    cv1.put(key, value64);
+
+    // Double
+    double dvalue = 3.5;
+    key = "double";
+    cv1.put(key, dvalue);
+
+    // BLOB
+    uint32_t data_len = 45;
+    char* src = new char[data_len];
+
+    memset(src, '$', data_len);
+    key = "data";
+    cv1.put(key, data_len, src);
+
+
+    delete[] src;
+
+    // STRING
+    std::string strVal = "whoo";
+    key = "string";
+    cv1.put(key, strVal);
+
+    // BOOL
+    bool mefalse = false;
+    key = "bool";
+    cv1.put(key, mefalse);
+
+    ContentValue cv2(cv1);
+
+    CHECK(cv1 == cv2);
+
+    cv1.clear();
+    cv2.clear();
+
+    REPORT("testCopyConstructor()");
+}
+
+
+void testClear(){
+
+    ContentValue cv1;
+
+    // INT 32
+    int value32 = 1;
+    std::string key = "one";
+
+    cv1.put(key, value32);
+
+    // INT 64
+    int64_t value64 = 423425242;
+    key = "int64";
+    cv1.put(key, value64);
+
+    // Double
+    double dvalue = 3.5;
+    key = "double";
+    cv1.put(key, dvalue);
+
+    // BLOB
+    uint32_t data_len = 45;
+    char* src = new char[data_len];
+
+    memset(src, '$', data_len);
+    key = "data";
+    cv1.put(key, data_len, src);
+
+
+    delete[] src;
+
+    // STRING
+    std::string strVal = "whoo";
+    key = "string";
+    cv1.put(key, strVal);
+
+    // BOOL
+    bool mefalse = false;
+    key = "bool";
+    cv1.put(key, mefalse);
+
+    std::map<std::string, uint8_t> ktMap;
+
+    cv1.getKeyTypeMap(ktMap);
+    CHECK(ktMap.size() > 0);
+
+    cv1.clear();
+
+    cv1.getKeyTypeMap(ktMap);
+    CHECK(ktMap.size() == 0);
+
+    REPORT("testClear()");
+
+}

libretroshare/src/tests/util/testretrocursor.cpp

@@ -0,0 +1,127 @@
+
+#include <iostream>
+#include <memory.h>
+#include <sstream>
+#include <time.h>
+
+#include "retrodb.h"
+#include "utest.h"
+
+
+#define INT32_KEY "day"
+#define INT64_KEY "pub_key"
+#define DATA_KEY "data"
+#define DOUBLE_KEY "a_double"
+#define STRING_KEY "a_string"
+#define BOOL_KEY "a_bool"
+
+#define DB_FILE_NAME "RetroDb"
+
+static RetroDb* mDb = NULL;
+
+void createRetroDb();
+void destroyRetroDb();
+
+INITTEST();
+// mainly test ability to move up and down result set
+
+
+void testTraverseResult();
+
+
+int main() {
+
+    testTraverseResult();
+
+    FINALREPORT("RETRO CURSOR TEST");
+    return 0;
+}
+
+
+
+void createRetroDb(){
+
+    if(mDb)
+        destroyRetroDb();
+
+
+    remove(DB_FILE_NAME); // account for interrupted tests
+    mDb = new RetroDb(DB_FILE_NAME, RetroDb::OPEN_READWRITE_CREATE);
+}
+
+void destroyRetroDb(){
+    if(mDb == NULL)
+        return;
+
+    mDb->closeDb();
+    delete mDb;
+    mDb = NULL;
+    int rc = remove(DB_FILE_NAME);
+    std::cerr << "remove code: " << rc << std::endl;
+
+    if(rc !=0){
+        perror("Could not delete db: ");
+
+    }
+}
+
+
+void testTraverseResult(){
+
+
+    createRetroDb();
+
+    bool statementExecuted = mDb->execSQL("CREATE TABLE retroDB(day INTEGER PRIMARY KEY ASC, pub_key INT, data BLOB, a_double REAL, a_string VARCHAR(255), a_bool INT);");
+    CHECK(statementExecuted);
+
+    ContentValue cv;
+
+    cv.put(INT32_KEY, (int32_t)20);
+    int64_t large_num = 32432242344423;
+    cv.put(INT64_KEY, large_num);
+    std::string str = "3dajaojaljfacjlaf£%£^%\"%\"%$";
+    const char* data = str.data();
+    int size = str.size();
+    cv.put(DATA_KEY, size, data);
+    cv.put(DOUBLE_KEY, 3.14);
+    cv.put(STRING_KEY, "hello precious");
+    cv.put(BOOL_KEY, false);
+
+    bool insertExecuted = mDb->sqlInsert("retroDB", "", cv);
+    cv.put(INT32_KEY, (int32_t)21);
+    insertExecuted &= mDb->sqlInsert("retroDB", "", cv);
+    cv.put(INT32_KEY, (int32_t)2);
+    insertExecuted &= mDb->sqlInsert("retroDB", "", cv);
+    cv.put(INT32_KEY, (int32_t)204);
+    insertExecuted &= mDb->sqlInsert("retroDB", "", cv);
+    cv.put(INT32_KEY, (int32_t)22);
+    insertExecuted &= mDb->sqlInsert("retroDB", "", cv);
+
+    CHECK(insertExecuted);
+
+    std::list<std::string> columns;
+    columns.push_back(INT32_KEY); columns.push_back(INT64_KEY); columns.push_back(DOUBLE_KEY);
+    columns.push_back(DATA_KEY); columns.push_back(BOOL_KEY); columns.push_back(STRING_KEY);
+    std::string orderBy = std::string(INT32_KEY) + " ASC";
+    RetroCursor* cursor = mDb->sqlQuery("retroDB", columns, "", orderBy);
+
+    CHECK(cursor->getResultCount() == 5);
+    cursor->moveToFirst();
+    CHECK(cursor->getPosition() == 0);
+
+    cursor->moveToLast();
+    CHECK(cursor->getPosition() == cursor->getResultCount());
+
+    cursor->moveToFirst();
+
+    CHECK(cursor->getInt32(0) == 2);
+    cursor->moveToNext();
+    cursor->moveToNext();
+
+    CHECK(cursor->getInt32(0) == 21);
+
+    delete cursor;
+
+    REPORT("testTraverseResult()");
+    destroyRetroDb();
+}

libretroshare/src/tests/util/testretrodb.cpp

@@ -0,0 +1,352 @@
+
+#include <iostream>
+#include <memory.h>
+#include <sstream>
+#include <time.h>
+
+#include "retrodb.h"
+#include "utest.h"
+
+
+#define DAY_KEY "day"
+#define PUB_KEY "pub_key"
+#define DATA_KEY "data"
+#define DOUBLE_KEY "a_double"
+
+#define DB_FILE_NAME "RetroDb"
+
+static RetroDb* mDb = NULL;
+
+void createRetroDb();
+void destroyRetroDb();
+
+void testSqlExec();
+void testSqlUpdate();
+void testSqlInsert();
+void testSqlDelete();
+void testSqlQuery();
+void testBinaryInsertion();
+
+INITTEST();
+
+int main(){
+
+    testSqlExec();
+    testBinaryInsertion();
+    testSqlInsert();
+    testSqlUpdate();
+    testSqlDelete();
+    testSqlQuery();
+
+    FINALREPORT("RETRO DB TEST");
+    return 0;
+}
+
+void createRetroDb(){
+
+    if(mDb)
+        destroyRetroDb();
+
+
+    remove(DB_FILE_NAME); // account for interrupted tests
+    mDb = new RetroDb(DB_FILE_NAME, RetroDb::OPEN_READWRITE_CREATE);
+}
+
+void destroyRetroDb(){
+    if(mDb == NULL)
+        return;
+
+    mDb->closeDb();
+    delete mDb;
+    mDb = NULL;
+    int rc = remove(DB_FILE_NAME);
+    std::cerr << "remove code: " << rc << std::endl;
+
+    if(rc !=0){
+        perror("Could not delete db: ");
+
+    }
+}
+
+void testSqlExec(){
+
+      createRetroDb();
+
+    // create simple table
+    bool statementExecuted = mDb->execSQL("CREATE TABLE retroDB(day INTEGER PRIMARY KEY ASC, pub_key, data);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(day, pub_key, data) VALUES(1,2,3);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(day, pub_key, data) VALUES(3,4525624,4524);");
+
+    CHECK(statementExecuted);
+
+    // now check if you can retrieve records
+    std::list<std::string> columns;
+    columns.push_back("day");
+    columns.push_back("pub_key");
+    columns.push_back("data");
+
+    std::string selection, orderBy;
+    RetroCursor* c = mDb->sqlQuery("retroDB", columns, selection, orderBy);
+
+
+    CHECK(c->getResultCount() == 2);
+
+    // got to first record
+    c->moveToFirst();
+    int32_t first =c->getInt32(0), second = c->getInt32(1),
+    third = c->getInt32(2);
+
+
+    CHECK(first == 1);
+    CHECK(second == 2);
+    CHECK(third == 3);
+
+    // get 2nd record
+    c->moveToNext();
+
+    first =c->getInt32(0), second = c->getInt32(1),
+    third = c->getInt32(2);
+
+    CHECK(first == 3);
+    CHECK(second == 4525624);
+    CHECK(third == 4524);
+
+    delete c;
+
+    REPORT("testSqlExec()");
+    destroyRetroDb();
+}
+
+void testBinaryInsertion(){
+
+    createRetroDb();
+    // create simple table
+    bool statementExecuted = mDb->execSQL("CREATE TABLE retroDB(day INTEGER PRIMARY KEY ASC, data BLOB);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(day, data) VALUES(1, 'dafadfad%$£%^£%%\"$R\"$\"$\"');");
+
+    // now check if you can retrieve records
+    std::list<std::string> columns;
+    columns.push_back("day");
+    columns.push_back("data");
+
+    std::string selection, orderBy;
+    RetroCursor* c = mDb->sqlQuery("retroDB", columns, selection, orderBy);
+
+    c->moveToFirst();
+    int first = c->getInt32(0);
+    uint32_t size;
+    const char* data = (const char*)c->getData(1, size);
+    std::string str = "dafadfad%$£%^£%%\"$R\"$\"$\"";
+    const char* data_comp = str.data();
+    bool binCompare = ok;
+
+    for(int i=0; i < 24 ; i++)
+        binCompare &= data[i] == data_comp[i];
+
+    CHECK(first == 1);
+    CHECK(binCompare);
+
+    delete c;
+    REPORT("testBinaryInsertion()");
+    destroyRetroDb();
+}
+
+
+
+
+void testSqlUpdate(){
+
+    createRetroDb();
+
+    bool statementExecuted = mDb->execSQL("CREATE TABLE retroDB(day INTEGER PRIMARY KEY ASC, data BLOB, peerId VARCHAR(255), time INT);");
+    CHECK(statementExecuted);
+
+    std::string data = "dadJOOodaodaoro20r2-0r20002ri02fgi3t0***";
+    ContentValue cv;
+
+    cv.put("day", (int32_t)20);
+    std::string peerid = "TheRetroSquad";
+    cv.put("peerId", peerid);
+    cv.put("data", data.size(), data.data());
+    int64_t now = time(NULL);
+    cv.put("time", now);
+    bool insertExecuted = mDb->sqlInsert("retroDB", "", cv);
+
+    CHECK(insertExecuted);
+
+    // now check entry is fine
+
+    std::list<std::string> columns;
+    columns.push_back("day"); columns.push_back("peerId"); columns.push_back("data"); columns.push_back("time");
+
+    RetroCursor* c = mDb->sqlQuery("retroDB", columns, "", "");
+
+    CHECK(c->getResultCount() == 1);
+    std::string result;
+    c->moveToFirst();
+    c->getString(1, result);
+
+    CHECK(result == peerid);
+
+    delete c;
+    c = NULL;
+
+    // now make an update and see if this is reflected
+
+    int64_t now_plus = now+203;
+    cv.put("time", now_plus);
+    bool update = mDb->sqlUpdate("retroDB", "", cv);
+    CHECK(update);
+    c = mDb->sqlQuery("retroDB", columns, "", "");
+    c->moveToFirst();
+    int64_t now_plus_compare = c->getDouble(3);
+
+    CHECK(now_plus_compare == now_plus);
+
+    // now attempt an update which should find no valid record
+
+
+    delete c;
+
+    REPORT("testSqlUpdate()");
+    destroyRetroDb();
+}
+
+void testSqlInsert(){
+
+    createRetroDb();
+
+    bool statementExecuted = mDb->execSQL("CREATE TABLE retroDB(day INTEGER PRIMARY KEY ASC, a_double DOUBLE, data BLOB);");
+    CHECK(statementExecuted);
+
+    ContentValue cv;
+
+    cv.put("day", (int32_t)20);
+    std::string str = "3dajaojaljfacjlaf£%£^%\"%\"%$";
+    const char* data = str.data();
+    int size = 27;
+    cv.put(DATA_KEY, size, data);
+    cv.put(DOUBLE_KEY, 3.14);
+    bool insertExecuted = mDb->sqlInsert("retroDB", "", cv);
+
+    std::list<std::string> columns;
+    columns.push_back("day"); columns.push_back("a_double"); columns.push_back("data");
+    RetroCursor* cursor = mDb->sqlQuery("retroDB", columns, "", "");
+
+    CHECK(cursor != NULL);
+
+    cursor->moveToFirst();
+
+    CHECK(20 == cursor->getInt32(0));
+    CHECK(3.14 == cursor->getDouble(1));
+
+    CHECK(insertExecuted);
+
+    delete cursor;
+    FINALREPORT("testSqlInsert()");
+    destroyRetroDb();
+}
+
+void testSqlQuery(){
+    //  test ordering of data and selection clause
+
+
+    createRetroDb();
+
+  // create simple table
+    bool statementExecuted = mDb->execSQL("CREATE TABLE retroDB(priority INTEGER PRIMARY KEY ASC, name VARCHAR(255), friends, games INTEGER);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(priority, name, friends, games) VALUES(4,'sammy',2,30);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(priority, name, friends, games) VALUES(2,'davy',6,9);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(priority, name, friends, games) VALUES(6,'pammy',2,4);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(priority, name, friends, games) VALUES(5,'tommy',3,4534);");
+    statementExecuted &= mDb->execSQL("INSERT INTO retroDB(priority, name, friends, games) VALUES(9,'jonny',3,44);");
+
+    CHECK(statementExecuted);
+
+    std::list<std::string> columns;
+    columns.push_back("name");
+    std::string selection = "games <= 31";
+    std::string orderBy = "priority DESC";
+
+
+    // output should be by name: pammy, sammy and davy
+    RetroCursor* cursor = mDb->sqlQuery("retroDB", columns,selection, orderBy);
+
+    cursor->moveToFirst();
+    std::string name;
+    cursor->getString(0,name);
+    CHECK(name == "pammy");
+
+    cursor->moveToNext();
+    cursor->getString(0,name);
+    CHECK(name == "sammy");
+
+    cursor->moveToNext();
+    cursor->getString(0,name);
+    CHECK(name == "davy");
+
+
+    delete cursor;
+    FINALREPORT("testSqlQuery()");
+    destroyRetroDb();
+    return;
+}
+
+void testSqlDelete(){
+
+    createRetroDb();
+
+    bool statementExecuted = mDb->execSQL("CREATE TABLE retroDB(day INTEGER PRIMARY KEY ASC, a_double DOUBLE, data BLOB);");
+    CHECK(statementExecuted);
+
+    ContentValue cv;
+
+    cv.put("day", (int32_t)20);
+    std::string str = "3dajaojaljfacjlaf£%£^%\"%\"%$";
+    const char* data = str.data();
+    uint32_t size = str.size();
+    cv.put(DATA_KEY, size, data);
+    cv.put(DOUBLE_KEY, 3.14);
+
+    // insert to records
+    bool insertExecuted = mDb->sqlInsert("retroDB", "", cv);
+    cv.put("day", (int32_t(5)));
+    CHECK(insertExecuted);
+    mDb->sqlInsert("retroDB", "", cv);
+
+
+    // now check that their are two records in the db
+    std::list<std::string> columns;
+    columns.push_back("day"); columns.push_back("a_double"); columns.push_back("data");
+    RetroCursor* cursor = mDb->sqlQuery("retroDB", columns, "", "");
+    CHECK(cursor->getResultCount() == 2);
+    delete cursor;
+    // now remove a record and search for the removed record, query should return no records
+    mDb->sqlDelete("retroDB", "day=5", "");
+
+
+    cursor = mDb->sqlQuery("retroDB", columns, "day=5", "");
+    CHECK(cursor->getResultCount() == 0);
+    delete cursor;
+
+    // now check for the remaining record
+    cursor = mDb->sqlQuery("retroDB", columns, "day=20", "");
+    CHECK(cursor->getResultCount() == 1);
+    cursor->moveToFirst();
+
+    // verify there is no data corruption
+    const char* data_comp1 = (const char*)cursor->getData(2, size);
+    const char* data_comp2 = str.data();
+
+    bool binCompare = true;
+
+    for(int i=0; i < str.size() ; i++)
+        binCompare &= data_comp1[i] == data_comp2[i];
+
+    CHECK(binCompare);
+
+    delete cursor;
+
+    FINALREPORT("testSqlDelete()");
+    destroyRetroDb();
+}

libretroshare/src/util/retrodb.cc

@@ -0,0 +1,990 @@
+
+/*
+ * RetroShare : RetroDb functionality
+ *
+ * Copyright 2012 Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include <iostream>
+#include <sstream>
+#include <memory.h>
+#include <time.h>
+#include <inttypes.h>
+
+#include "retrodb.h"
+
+//#define RETRODB_DEBUG
+
+void free_blob(void* dat){
+
+    char* c = (char*) dat;
+    delete[] c;
+    dat = NULL;
+
+}
+
+const uint8_t ContentValue::BOOL_TYPE  = 1;
+const uint8_t ContentValue::DATA_TYPE = 2;
+const uint8_t ContentValue::STRING_TYPE = 3;
+const uint8_t ContentValue::DOUBLE_TYPE = 4;
+const uint8_t ContentValue::INT32_TYPE = 5;
+const uint8_t ContentValue::INT64_TYPE = 6;
+
+const int RetroDb::OPEN_READONLY = SQLITE_OPEN_READONLY;
+const int RetroDb::OPEN_READWRITE = SQLITE_OPEN_READWRITE;
+const int RetroDb::OPEN_READWRITE_CREATE = SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE;
+
+RetroDb::RetroDb(const std::string &dbPath, int flags) : mDb(NULL) {
+
+    int rc = sqlite3_open_v2(dbPath.c_str(), &mDb, flags, NULL);
+
+    if(rc){
+        std::cerr << "Can't open database, Error code: " <<  sqlite3_errmsg(mDb)
+                  << std::endl;
+        sqlite3_close(mDb);
+     }
+}
+
+
+RetroDb::~RetroDb(){
+
+    if(mDb)
+        sqlite3_close(mDb);
+}
+
+void RetroDb::closeDb(){
+
+    int rc;
+
+    if(mDb)
+        rc  = sqlite3_close(mDb);
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "RetroDb::closeDb(): Error code on close: " << rc << std::endl;
+#endif
+
+}
+
+#define TIME_LIMIT 3
+
+bool RetroDb::execSQL(const std::string &query){
+
+    // prepare statement
+    sqlite3_stmt* stm = NULL;
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "Query: " << query << std::endl;
+#endif
+
+    int rc = sqlite3_prepare_v2(mDb, query.c_str(), query.length(), &stm, NULL);
+
+    // check if there are any errors
+    if(rc != SQLITE_OK){
+        std::cerr << "RetroDb::execSQL(): Error preparing statement\n";
+        std::cerr << "Error code: " <<  sqlite3_errmsg(mDb)
+                  << std::endl;
+        return false;
+    }
+
+
+    uint32_t delta = 3;
+    time_t stamp = time(NULL), now = 0;
+    bool timeOut = false, ok = false;
+
+    while(!timeOut){
+
+        rc = sqlite3_step(stm);
+
+        if(rc == SQLITE_DONE){
+            ok = true;
+            break;
+        }
+
+        if(rc != SQLITE_BUSY){
+            ok = false;
+            break;
+        }
+
+        now = time(NULL);
+        delta = stamp - now;
+
+        if(delta > TIME_LIMIT){
+            ok = false;
+            timeOut = true;
+        }
+        // TODO add sleep so not to waste
+        // precious cycles
+    }
+
+    if(!ok){
+
+        if(rc == SQLITE_BUSY){
+            std::cerr << "RetroDb::execSQL()\n" ;
+            std::cerr << "SQL timed out!" << std::endl;
+        }else{
+            std::cerr << "RetroDb::execSQL(): Error executing statement (code: " << rc << ")\n";
+            std::cerr << "Sqlite Error msg: " <<  sqlite3_errmsg(mDb)
+                      << std::endl;
+        }
+    }
+
+    // finalise statement or else db cannot be closed
+    sqlite3_finalize(stm);
+    return ok;
+}
+
+RetroCursor* RetroDb::sqlQuery(const std::string& tableName, const std::list<std::string>& columns,
+                               const std::string& selection, const std::string& orderBy){
+
+    if(tableName.empty() || columns.empty()){
+        std::cerr << "RetroDb::sqlQuery(): No table or columns given" << std::endl;
+        return NULL;
+    }
+
+    std::string columnSelection; // the column names to return
+    sqlite3_stmt* stmt = NULL;
+    std::list<std::string>::const_iterator it = columns.begin();
+
+    for(; it != columns.end(); it++){
+        columnSelection += *it;
+
+        it++;
+        if(it != columns.end())
+            columnSelection += ",";
+        it--;
+    }
+
+    // construct query
+    // SELECT columnSelection FROM tableName WHERE selection
+    std::string sqlQuery = "SELECT " + columnSelection + " FROM " +
+                           tableName;
+
+    // add selection clause if present
+    if(!selection.empty())
+        sqlQuery += " WHERE " + selection;
+
+
+    // add 'order by' clause if present
+    if(!orderBy.empty())
+        sqlQuery += " ORDER BY " + orderBy + ";";
+    else
+        sqlQuery += ";";
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "RetroDb::sqlQuery(): " << sqlQuery << std::endl;
+#endif
+
+    sqlite3_prepare_v2(mDb, sqlQuery.c_str(), sqlQuery.length(), &stmt, NULL);
+    return (new RetroCursor(stmt));
+}
+
+bool RetroDb::isOpen() const {
+    return (mDb==NULL ? false : true);
+}
+
+bool RetroDb::sqlInsert(const std::string &table, const std::string& nullColumnHack, const ContentValue &cv){
+
+    std::map<std::string, uint8_t> keyTypeMap;
+    cv.getKeyTypeMap(keyTypeMap);
+    std::map<std::string, uint8_t>::iterator mit = keyTypeMap.begin();
+
+    // build columns part of insertion
+    std::string qColumns = table + "(";
+
+    for(; mit != keyTypeMap.end(); mit++){
+
+        qColumns += mit->first;
+
+        mit++;
+
+        // add comma if more columns left
+        if(mit == keyTypeMap.end())
+            qColumns += ")"; // close bracket if at end
+        else
+             qColumns += ",";
+
+        mit--;
+    }
+
+    // build values part of insertion
+    std::string qValues = "VALUES(";
+    std::ostringstream oStrStream;
+    uint32_t index = 0;
+    std::list<RetroDbBlob> blobL;
+
+    for(mit=keyTypeMap.begin(); mit!=keyTypeMap.end(); mit++){
+
+        uint8_t type = mit->second;
+        std::string key = mit->first;
+
+        switch(type){
+
+        case ContentValue::BOOL_TYPE:
+            {
+                bool value;
+                cv.getAsBool(key, value);
+                oStrStream << value;
+                qValues += oStrStream.str();
+                break;
+            }
+        case ContentValue::DOUBLE_TYPE:
+            {
+                double value;
+                cv.getAsDouble(key, value);
+                oStrStream << value;
+                qValues += oStrStream.str();
+                break;
+            }
+        case ContentValue::DATA_TYPE:
+            {
+                char* value;
+                uint32_t len;
+                cv.getAsData(key, len, value);
+                RetroDbBlob b;
+                b.data = value;
+                b.length = len;
+                b.index = ++index;
+                blobL.push_back(b);
+                qValues += "?"; // parameter
+                break;
+            }
+        case ContentValue::STRING_TYPE:
+            {
+                std::string value;
+                cv.getAsString(key, value);
+                qValues += "'" + value +"'";
+                break;
+            }
+        case ContentValue::INT32_TYPE:
+            {
+                int32_t value;
+                cv.getAsInt32(key, value);
+                oStrStream << value;
+                qValues += oStrStream.str();
+                break;
+            }
+        case ContentValue::INT64_TYPE:
+            {
+                int64_t value;
+                cv.getAsInt64(key, value);
+                oStrStream << value;
+                qValues += oStrStream.str();
+                break;
+            }
+        }
+
+        mit++;
+        if(mit != keyTypeMap.end()){ // add comma if more columns left
+            qValues += ",";
+        }
+        else{ // at end close brackets
+             qValues += ");";
+         }
+        mit--;
+
+        // clear stream strings
+        oStrStream.str("");
+    }
+
+
+    // complete insertion query
+    std::string sqlQuery = "INSERT INTO " + qColumns + " " + qValues;
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "RetroDb::sqlInsert(): " << sqlQuery << std::endl;
+#endif
+
+    // execute query
+    execSQL_bind_blobs(sqlQuery, blobL);
+    return true;
+}
+
+bool RetroDb::execSQL_bind_blobs(const std::string &query, std::list<RetroDbBlob> &blobs){
+
+    // prepare statement
+    sqlite3_stmt* stm = NULL;
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "Query: " << query << std::endl;
+#endif
+
+    int rc = sqlite3_prepare_v2(mDb, query.c_str(), query.length(), &stm, NULL);
+
+    // check if there are any errors
+    if(rc != SQLITE_OK){
+        std::cerr << "RetroDb::execSQL(): Error preparing statement\n";
+        std::cerr << "Error code: " <<  sqlite3_errmsg(mDb)
+                  << std::endl;
+        return false;
+    }
+
+    std::list<RetroDbBlob>::iterator lit = blobs.begin();
+
+    for(; lit != blobs.end(); lit++){
+        const RetroDbBlob& b = *lit;
+        sqlite3_bind_blob(stm, b.index, b.data, b.length, NULL);
+    }
+
+    uint32_t delta = 3;
+    time_t stamp = time(NULL), now = 0;
+    bool timeOut = false, ok = false;
+
+    while(!timeOut){
+
+        rc = sqlite3_step(stm);
+
+        if(rc == SQLITE_DONE){
+            ok = true;
+            break;
+        }
+
+        if(rc != SQLITE_BUSY){
+            ok = false;
+            break;
+        }
+
+        now = time(NULL);
+        delta = stamp - now;
+
+        if(delta > TIME_LIMIT){
+            ok = false;
+            timeOut = true;
+        }
+        // TODO add sleep so not to waste
+        // precious cycles
+    }
+
+    if(!ok){
+
+        if(rc == SQLITE_BUSY){
+            std::cerr << "RetroDb::execSQL()\n" ;
+            std::cerr << "SQL timed out!" << std::endl;
+        }else{
+            std::cerr << "RetroDb::execSQL(): Error executing statement (code: " << rc << ")\n";
+            std::cerr << "Sqlite Error msg: " <<  sqlite3_errmsg(mDb)
+                      << std::endl;
+        }
+    }
+
+    // finalise statement or else db cannot be closed
+    sqlite3_finalize(stm);
+    return ok;
+}
+
+bool RetroDb::sqlDelete(const std::string &tableName, const std::string &whereClause, const std::string &whereArgs){
+
+    std::string sqlQuery = "DELETE FROM " + tableName;
+
+    if(!whereClause.empty()){
+        sqlQuery += " WHERE " + whereClause + ";";
+    }else
+        sqlQuery += ";";
+
+    return execSQL(sqlQuery);
+}
+
+
+bool RetroDb::sqlUpdate(const std::string &tableName, std::string whereClause, const ContentValue& cv){
+
+    std::string sqlQuery = "UPDATE " + tableName + " SET ";
+
+
+    std::map<std::string, uint8_t> keyTypeMap;
+    std::map<std::string, uint8_t>::iterator mit;
+    cv.getKeyTypeMap(keyTypeMap);
+
+    // build SET part of update
+    std::string qValues = "";
+    std::ostringstream oStrStream;
+
+    for(mit=keyTypeMap.begin(); mit!=keyTypeMap.end(); mit++){
+
+        uint8_t type = mit->second;
+        std::string key = mit->first;
+
+        switch(type){
+
+        case ContentValue::BOOL_TYPE:
+            {
+                bool value;
+                cv.getAsBool(key, value);
+                oStrStream << value;
+                qValues += key + "='" + oStrStream.str();
+                break;
+            }
+        case ContentValue::DOUBLE_TYPE:
+            {
+                double value;
+                cv.getAsDouble(key, value);
+                oStrStream << value;
+                qValues += key + "='" + oStrStream.str();
+                break;
+            }
+        case ContentValue::DATA_TYPE:
+            {
+                char* value;
+                uint32_t len;
+                cv.getAsData(key, len, value);
+                oStrStream.write(value, len);
+                qValues += key + "='" + oStrStream.str() + "' ";
+                break;
+            }
+        case ContentValue::STRING_TYPE:
+            {
+                std::string value;
+                cv.getAsString(key, value);
+                qValues += key + "='" + value + "' ";
+                break;
+            }
+        case ContentValue::INT32_TYPE:
+            {
+                int32_t value;
+                cv.getAsInt32(key, value);
+                oStrStream << value;
+                qValues += key + "='" + oStrStream.str() + "' ";
+                break;
+            }
+        case ContentValue::INT64_TYPE:
+            {
+                int64_t value;
+                cv.getAsInt64(key, value);
+                oStrStream << value;
+                qValues += key + "='" + oStrStream.str() + "' ";
+                break;
+            }
+        }
+
+        mit++;
+        if(mit != keyTypeMap.end()){ // add comma if more columns left
+            qValues += ",";
+        }
+        mit--;
+
+        // clear stream strings
+        oStrStream.str("");
+    }
+
+    if(qValues.empty())
+        return false;
+    else
+        sqlQuery += qValues;
+
+    // complete update
+    if(!whereClause.empty()){
+        sqlQuery += " WHERE " +  whereClause + ";";
+    }
+    else{
+        sqlQuery += ";";
+    }
+
+    // execute query
+    return execSQL(sqlQuery);
+
+}
+
+
+
+
+
+/********************** RetroCursor ************************/
+
+RetroCursor::RetroCursor(sqlite3_stmt *stmt)
+    : mStmt(NULL), mCount(0), mPosCounter(0) {
+
+     open(stmt);
+}
+
+RetroCursor::~RetroCursor(){
+
+    // finalise statement
+    if(mStmt){
+        sqlite3_finalize(mStmt);
+    }
+}
+
+bool RetroCursor::moveToFirst(){
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "RetroCursor::moveToFirst()\n";
+#endif
+
+    if(!isOpen())
+        return false;
+
+    // reset statement
+    int rc = sqlite3_reset(mStmt);
+
+    if(rc != SQLITE_OK){
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "Error code: " << rc << std::endl;
+#endif
+
+        return false;
+    }
+
+    rc = sqlite3_step(mStmt);
+
+    if(rc == SQLITE_ROW){
+        mPosCounter = 0;
+        return true;
+    }
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "Error code: " << rc << std::endl;
+#endif
+
+    return false;
+}
+
+bool RetroCursor::moveToLast(){
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "RetroCursor::moveToLast()\n";
+#endif
+
+    if(!isOpen())
+        return -1;
+
+    // go to begining
+    int rc = sqlite3_reset(mStmt);
+
+    if(rc != SQLITE_OK)
+        return false;
+
+    rc = sqlite3_step(mStmt);
+
+    while(rc == SQLITE_ROW){
+        rc = sqlite3_step(mStmt);
+    }
+
+    if(rc != SQLITE_DONE){
+        std::cerr << "Error executing statement (code: " << rc << ")\n"
+                  << std::endl;
+        return false;
+
+    }else{
+        mPosCounter = mCount;
+        return true;
+    }
+}
+
+int RetroCursor::getResultCount() const {
+
+    if(isOpen())
+        return mCount;
+    else
+        return -1;
+}
+
+bool RetroCursor::isOpen() const {
+    return !(mStmt == NULL);
+}
+
+bool RetroCursor::close(){
+
+    if(!isOpen())
+        return false;
+
+
+    int rc = sqlite3_finalize(mStmt);
+    mStmt = NULL;
+    mPosCounter = 0;
+    mCount = 0;
+
+    return (rc == SQLITE_OK);
+}
+
+bool RetroCursor::open(sqlite3_stmt *stm){
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "RetroCursor::open() \n";
+#endif
+
+    if(isOpen())
+        close();
+
+    mStmt = stm;
+
+    // ensure statement is valid
+    int rc = sqlite3_reset(mStmt);
+
+    if(rc == SQLITE_OK){
+
+        while((rc = sqlite3_step(mStmt)) == SQLITE_ROW)
+            mCount++;
+
+        sqlite3_reset(mStmt);
+
+        return true;
+    }
+    else{
+        std::cerr << "Error Opening cursor (code: " << rc << ")\n";
+        close();
+        return false;
+    }
+
+}
+
+bool RetroCursor::moveToNext(){
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "RetroCursor::moveToNext()\n";
+#endif
+
+    if(!isOpen())
+        return false;
+
+    int rc = sqlite3_step(mStmt);
+
+    if(rc == SQLITE_ROW){
+        mPosCounter++;
+        return true;
+
+    }else if(rc == SQLITE_DONE){ // no more results
+        return false;
+    }
+    else if(rc == SQLITE_BUSY){ // should not enter here
+        std::cerr << "RetroDb::moveToNext()\n" ;
+        std::cerr << "Busy!, possible multiple accesses to Db" << std::endl
+                  << "serious error";
+
+        return false;
+
+    }else{
+        std::cerr << "Error executing statement (code: " << rc << ")\n";
+        return false;
+    }
+}
+
+
+int32_t RetroCursor::getPosition() const {
+
+    if(isOpen())
+        return mPosCounter;
+    else
+        return -1;
+}
+
+
+int32_t RetroCursor::getInt32(int columnIndex){
+    return sqlite3_column_int(mStmt, columnIndex);
+}
+
+int64_t RetroCursor::getInt64(int columnIndex){
+    return sqlite3_column_int64(mStmt, columnIndex);
+}
+
+bool RetroCursor::getBool(int columnIndex){
+    return sqlite3_column_int(mStmt, columnIndex);
+}
+
+double RetroCursor::getDouble(int columnIndex){
+    return sqlite3_column_double(mStmt, columnIndex);
+}
+
+void RetroCursor::getString(int columnIndex, std::string &str){
+    char* raw_str = (char*)sqlite3_column_text(mStmt, columnIndex);
+    if(raw_str != NULL)
+        str.assign(raw_str);
+}
+
+const void* RetroCursor::getData(int columnIndex, uint32_t &datSize){
+
+    const void* val = sqlite3_column_blob(mStmt, columnIndex);
+    datSize = sqlite3_column_bytes(mStmt, columnIndex);
+
+    return val;
+}
+
+
+
+
+/**************** content value implementation ******************/
+
+typedef std::pair<std::string, uint8_t> KeyTypePair;
+
+ContentValue::ContentValue(){
+
+}
+
+ContentValue::~ContentValue(){
+    // release resources held in data
+    clearData();
+}
+
+ContentValue::ContentValue(ContentValue &from){
+
+    std::map<std::string, uint8_t> keyTypeMap;
+    from.getKeyTypeMap(keyTypeMap);
+    std::map<std::string, uint8_t>::const_iterator cit =
+            keyTypeMap.begin();
+
+    uint8_t type = 0;
+    std::string currKey;
+    std::string val = "";
+    char *src = NULL;
+    uint32_t data_len = 0;
+
+    for(; cit != keyTypeMap.end(); cit++){
+
+        type = cit->second;
+        currKey = cit->first;
+
+        switch(type){
+
+        case INT32_TYPE:
+            {
+                int32_t value;
+                from.getAsInt32(currKey, value);
+                put(currKey, value);
+                break;
+            }
+        case INT64_TYPE:
+            {
+                int64_t value;
+                from.getAsInt64(currKey, value);
+                put(currKey, value);
+                break;
+            }
+        case STRING_TYPE:
+            {
+                from.getAsString(currKey, val);
+                put(currKey, val);
+                break;
+            }
+        case BOOL_TYPE:
+            {
+                bool value;
+                from.getAsBool(currKey, value);
+                put(currKey, value);
+                break;
+            }
+        case DATA_TYPE:
+            {
+                from.getAsData(currKey, data_len, src);
+                put(currKey, data_len, src);
+                break;
+            }
+        case DOUBLE_TYPE:
+            double value;
+            from.getAsDouble(currKey, value);
+            put(currKey, value);
+            break;
+        default:
+            std::cerr << "ContentValue::ContentValue(ContentValue &from):"
+                    << "Error! Unrecognised data type!" << std::endl;
+        }
+    }
+}
+
+void ContentValue::put(const std::string &key, bool value){
+
+    if(mKvSet.find(key) != mKvSet.end())
+        removeKeyValue(key);
+
+    mKvSet.insert(KeyTypePair(key, BOOL_TYPE));
+    mKvBool.insert(std::pair<std::string, bool>(key, value));
+}
+
+void ContentValue::put(const std::string &key, const std::string &value){
+
+    if(mKvSet.find(key) != mKvSet.end())
+        removeKeyValue(key);
+
+    mKvSet.insert(KeyTypePair(key, STRING_TYPE));
+    mKvString.insert(std::pair<std::string, std::string>(key, value));
+}
+
+void ContentValue::put(const std::string &key, double value){
+
+    if(mKvSet.find(key) != mKvSet.end())
+        removeKeyValue(key);
+
+    mKvSet.insert(KeyTypePair(key,DOUBLE_TYPE));
+    mKvDouble.insert(std::pair<std::string, double>(key, value));
+}
+
+void ContentValue::put(const std::string &key, int32_t value){
+
+    if(mKvSet.find(key) != mKvSet.end())
+        removeKeyValue(key);
+
+    mKvSet.insert(KeyTypePair(key, INT32_TYPE));
+    mKvInt32.insert(std::pair<std::string, int32_t>(key, value));
+}
+
+void ContentValue::put(const std::string &key, int64_t value){
+
+    if(mKvSet.find(key) != mKvSet.end())
+        removeKeyValue(key);
+
+    mKvSet.insert(KeyTypePair(key, INT64_TYPE));
+    mKvInt64.insert(std::pair<std::string, int64_t>(key, value));
+}
+
+void ContentValue::put(const std::string &key, uint32_t len, const char* value){
+
+
+    // release memory from old key value if key
+    // exists
+    if(mKvSet.find(key) != mKvSet.end()) {
+        removeKeyValue(key);
+    }
+
+    mKvSet.insert(KeyTypePair(key, DATA_TYPE));
+    char* dest = NULL;
+
+    // len is zero then just put a NULL entry
+    if(len != 0){
+        dest  = new char[len];
+        memcpy(dest, value, len);
+    }
+
+    mKvData.insert(std::pair<std::string, std::pair<uint32_t, char*> >
+                   (key, std::pair<uint32_t, char*>(len, dest)));
+}
+
+bool ContentValue::getAsBool(const std::string &key, bool& value) const{
+
+    std::map<std::string, bool>::const_iterator it;
+    if((it = mKvBool.find(key)) == mKvBool.end())
+        return false;
+
+  value = it->second;
+  return true;
+}
+
+bool ContentValue::getAsInt32(const std::string &key, int32_t& value) const{
+
+    std::map<std::string, int32_t>::const_iterator it;
+    if((it = mKvInt32.find(key)) == mKvInt32.end())
+        return false;
+
+    value = it->second;
+    return true;
+}
+
+bool ContentValue::getAsInt64(const std::string &key, int64_t& value) const{
+
+    std::map<std::string, int64_t>::const_iterator it;
+    if((it = mKvInt64.find(key)) == mKvInt64.end())
+        return false;
+
+    value = it->second;
+    return true;
+}
+
+bool ContentValue::getAsString(const std::string &key, std::string &value) const{
+
+    std::map<std::string, std::string>::const_iterator it;
+    if((it = mKvString.find(key)) == mKvString.end())
+        return false;
+
+    value = it->second;
+    return true;
+}
+
+bool ContentValue::getAsData(const std::string& key, uint32_t &len, char*& value) const{
+
+    std::map<std::string, std::pair<uint32_t, char*> >::const_iterator it;
+    if((it = mKvData.find(key)) == mKvData.end())
+        return false;
+
+    const std::pair<uint32_t, char*> &kvRef = it->second;
+
+    len = kvRef.first;
+    value = kvRef.second;
+    return true;
+}
+
+bool ContentValue::getAsDouble(const std::string &key, double& value) const{
+
+    std::map<std::string, double>::const_iterator it;
+    if((it = mKvDouble.find(key)) == mKvDouble.end())
+        return false;
+
+    value = it->second;
+    return true;
+}
+
+bool ContentValue::removeKeyValue(const std::string &key){
+
+    std::map<std::string, uint8_t>::iterator mit;
+
+    if((mit = mKvSet.find(key)) == mKvSet.end())
+        return false;
+
+    if(mit->second == BOOL_TYPE)
+        mKvBool.erase(key);
+
+    if(mit->second == INT64_TYPE)
+        mKvInt64.erase(key);
+
+    if(mit->second == DATA_TYPE){
+        delete[] (mKvData[key].second);
+        mKvData.erase(key);
+    }
+
+    if(mit->second == DOUBLE_TYPE)
+        mKvDouble.erase(key);
+
+    if(mit->second == STRING_TYPE)
+        mKvString.erase(key);
+
+    if(mit->second == INT32_TYPE)
+        mKvInt32.erase(key);
+
+
+    mKvSet.erase(key);
+    return true;
+}
+
+
+void ContentValue::getKeyTypeMap(std::map<std::string, uint8_t> &keySet) const {
+    keySet = mKvSet;
+}
+
+void ContentValue::clear(){
+    mKvSet.clear();
+    mKvBool.clear();
+    mKvDouble.clear();
+    mKvString.clear();
+    mKvInt32.clear();
+    mKvInt64.clear();
+    clearData();
+}
+
+void ContentValue::clearData(){
+
+    std::map<std::string, std::pair<uint32_t, char*> >::iterator
+            mit = mKvData.begin();
+
+    for(; mit != mKvData.end(); mit++){
+
+        if(mit->second.first != 0)
+            delete[] (mit->second.second);
+    }
+
+    mKvData.clear();
+}
+
+
+

libretroshare/src/util/retrodb.h

@@ -0,0 +1,444 @@
+#ifndef RSSQLITE_H
+#define RSSQLITE_H
+
+/*
+ * RetroShare : RetroDb functionality
+ *
+ * Copyright 2012 Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "sqlite3.h"
+
+#include <string>
+#include <set>
+#include <list>
+#include <map>
+
+
+class ContentValue;
+class RetroCursor;
+
+/*!
+ * RetroDb provide a means for Retroshare's core and \n
+ * services to maintain an easy to use random access file via a database \n
+ * It models itself after android's sqlite functionality \n
+ * This is essentially unashamedly a clone of Android's SQLiteDatabase interface
+ */
+class RetroDb
+{
+public:
+
+    /*!
+     * @param dbPath path to data base file
+     * @param flags determine where to open read only or read/write
+     */
+    RetroDb(const std::string& dbPath, int flags);
+
+    /*!
+     * closes db if it is not already closed
+     */
+    ~RetroDb();
+
+    /*!
+     * opens sqlite data base
+     * @param dbPath
+     * @param flags
+     * @return false if failed to open, true otherwise
+     */
+    bool openDb(const std::string& dbPath, int flags = OPEN_READONLY);
+
+    /*!
+     * close the database
+     */
+    void closeDb();
+
+    /*!
+     *
+     * @return false if database is not open, true otherwise
+     */
+    bool isOpen() const;
+
+    /* modifying db */
+public:
+
+
+
+    /*!
+     * To a make query which do not return a result \n
+     * below are the type of queries this method should be used for \n
+     * ALTER TABLE \n
+     * CREATE or DROP table / trigger / view / index / virtual table \n
+     * REINDEX \n
+     * RELEASE \n
+     * SAVEPOINT \n
+     * PRAGMA that returns no data \n
+     * @param query SQL query
+     * @return false if there was an sqlite error, true otherwise
+     */
+    bool execSQL(const std::string& query);
+
+    /*!
+     * inserts a row in a database table
+     * @param table table you want to insert content values into
+     * @param nullColumnHack  SQL doesn't allow inserting a completely \n
+     *        empty row without naming at least one column name
+     * @param cv hold entries to insert
+     * @return true if insertion successful, false otherwise
+     */
+    bool sqlInsert(const std::string& table,const  std::string& nullColumnHack, const ContentValue& cv);
+
+    /*!
+     * update row in a database table
+     * @param tableName the table on which to apply the UPDATE
+     * @param whereClause formatted as where statement without 'WHERE' itself
+     * @param cv Values used to replace current values in accessed record
+     * @return true if update was successful, false otherwise
+     */
+    bool sqlUpdate(const std::string& tableName, const std::string whereClause, const ContentValue& cv);
+
+    /*!
+     * Query the given table, returning a Cursor over the result set
+     * @param tableName the table name
+     * @param columns list columns that should be returned and their order (the list's order)
+     * @param selection  A filter declaring which rows to return, formatted as \n
+     *        an SQL WHERE clause (excluding the WHERE itself). Passing null will \n
+     *        return all rows for the given table.
+     * @param order the rows, formatted as an SQL ORDER BY clause (excluding the ORDER BY itself)
+     * @return cursor over result set, this allocated resource should be free'd after use \n
+     *         column order is in list order.
+     */
+    RetroCursor* sqlQuery(const std::string& tableName, const std::list<std::string>& columns,
+                          const std::string& selection, const std::string& orderBy);
+
+    /*!
+     * delete row in an sql table
+     * @param tableName the table on which to apply the DELETE
+     * @param whereClause formatted as where statement without 'WHERE' itself
+     * @return false
+     */
+    bool sqlDelete(const std::string& tableName, const std::string& whereClause, const std::string& whereArgs);
+
+    /*!
+     * TODO
+     * defragment database, should be done on databases if many modifications have occured
+     */
+    void vacuum();
+
+public:
+
+    static const int OPEN_READONLY;
+    static const int OPEN_READWRITE;
+    static const int OPEN_READWRITE_CREATE;
+
+private:
+
+    class RetroDbBlob{
+
+    public:
+
+        char* data;
+        uint32_t length;
+        uint32_t index;
+    };
+
+    bool execSQL_bind_blobs(const std::string &query, std::list<RetroDbBlob>& blobs);
+
+private:
+
+    sqlite3* mDb;
+
+};
+
+/*!
+ * Exposes result set from retrodb query
+ */
+class RetroCursor {
+
+public:
+
+    /*!
+     * Initialises a null cursor
+     * @warning cursor takes ownership of statement passed to it
+     */
+    RetroCursor(sqlite3_stmt*);
+
+    ~RetroCursor();
+
+    /*!
+     * move to first row of results
+     * @return false if no results
+     */
+    bool moveToFirst();
+
+    /*!
+     * move to next row of results
+     * @return false if no further results
+     */
+    bool moveToNext();
+
+    /*!
+     * move to last row of results
+     * @return false if no result, true otherwise
+     */
+    bool moveToLast();
+
+    /*!
+     * gets current position of cursor
+     * @return current position of cursor
+     */
+    int32_t getPosition() const;
+
+    /* data retrieval */
+
+    /*!
+     * @return true if cursor is open and active, false otherwise
+     */
+    bool isOpen() const;
+
+    /*!
+     * cursor is closed, statement used to open cursor is deleted
+     * @return false if error on close (was already closed, error occured)
+     */
+    bool close();
+
+    /*!
+     *
+     * @return -1 if cursor is in error, otherwise number of rows in result
+     */
+    int32_t getResultCount() const;
+
+    /*!
+     * Current statement is closed and discarded (finalised)
+     * before actual opening occurs
+     * @param stm statement to open cursor on
+     * @return true if cursor is successfully opened
+     */
+    bool open(sqlite3_stmt* stm);
+
+public:
+
+
+    /*!
+     * Returns the value of the requested column as a String.
+     * @param columnIndex the zero-based index of the target column.
+     * @return  the value of the column as 32 bit integer
+     */
+    int32_t getInt32(int columnIndex);
+
+    /*!
+     * Returns the value of the requested column as a String.
+     * @param columnIndex the zero-based index of the target column.
+     * @return the value of the column as 64 bit integer
+     */
+    int64_t getInt64(int columnIndex);
+
+    /*!
+     * Returns the value of the requested column as a String.
+     * @param columnIndex the zero-based index of the target column.
+     * @return  the value of the column as 64 bit float
+     */
+    double getDouble(int columnIndex);
+
+    /*!
+     * Returns the value of the requested column as a String.
+     * @param columnIndex the zero-based index of the target column.
+     * @return  the value of the column as bool
+     */
+    bool getBool(int columnIndex);
+
+    /*!
+     * Returns the value of the requested column as a String.
+     * @param columnIndex the zero-based index of the target column.
+     * @return the value of the column as a string
+     */
+    void getString(int columnIndex, std::string& str);
+
+    /*!
+     * Returns the value of the requested column as a String.
+     * data returned must be copied, as it is freed after RetroDb
+     * is closed or destroyed
+     * @param columnIndex the zero-based index of the target column.
+     * @return  the value of the column as pointer to raw data
+     */
+    const void* getData(int columnIndex, uint32_t& datSize);
+
+
+private:
+
+    sqlite3_stmt* mStmt;
+    int mCount; /// number of results
+    int mPosCounter;
+};
+
+
+/*!
+ * @brief Convenience container for making additions to databases
+ * This class provides a means of holding column values to insert into a database
+ */
+class ContentValue {
+
+public:
+
+    static const uint8_t INT32_TYPE;
+    static const uint8_t INT64_TYPE;
+    static const uint8_t DOUBLE_TYPE;
+    static const uint8_t STRING_TYPE;
+    static const uint8_t DATA_TYPE;
+    static const uint8_t BOOL_TYPE;
+
+    ContentValue();
+
+    /*!
+     * copy constructor that copys the key value set from another \n
+     * ContentValue object to this one
+     * makes a deep copy of raw data
+     * @param from ContentValue instance to copy key value set from
+     */
+    ContentValue(ContentValue& from);
+
+    /*!
+     *
+     *
+     *
+     */
+    ~ContentValue();
+
+    /*!
+     * Adds a value to the set
+     * @param key  the name of the value to put
+     * @param value  the data for the value to put
+     * @warning cast string literals explicitly as string, observed string literal \n
+     *          being casted to bool instead e.g. string("hello") rather than "hello"
+     */
+    void put(const std::string& key, const std::string& value);
+
+    /*!
+     * Adds a value to the set
+     * @param key  the name of the value to put
+     * @param value  the data for the value to put
+     */
+    void put(const std::string& key, bool value);
+
+    /*!
+     * Adds a value to the set
+     * @param key  the name of the value to put
+     * @param value  the data for the value to put
+     */
+    void put(const std::string& key, int64_t value);
+
+    /*!
+     * Adds a value to the set
+     * @param key  the name of the value to put
+     * @param value  the data for the value to put
+     */
+    void put(const std::string& key, int32_t value);
+
+    /*!
+     * Adds a value to the set
+     * @param key  the name of the value to put
+     * @param value  the data for the value to put
+     */
+    void put(const std::string& key, double value);
+
+    /*!
+     * Adds a value to the set
+     * @param key  the name of the value to put
+     * @param value  the data for the value to put
+     */
+    void put(const std::string& key, uint32_t len, const char* value);
+
+
+    /*!
+     * get value as 32 bit signed integer
+     * @param key the value to get
+     */
+    bool getAsInt32(const std::string& key, int32_t& value) const;
+
+    /*!
+     * get value as 64 bit signed integer
+     * @param key the value to get
+     */
+    bool getAsInt64(const std::string& key, int64_t& value) const;
+
+    /*!
+     * get value as bool
+     * @param key the value to get
+     */
+    bool getAsBool(const std::string& key, bool& value) const;
+
+    /*!
+     * get as value double
+     * @param key the value to get
+     */
+    bool getAsDouble(const std::string& key, double& value) const;
+
+    /*!
+     * get as value as string
+     * @param key the value to get
+     * @param value the data retrieved
+     */
+    bool getAsString(const std::string& key, std::string& value) const;
+
+    /*!
+     * get as value as raw data
+     * @warning Deep copy of data reference should be made, if this instance ContentValue \n
+     *          is destroyed pointer returned (value) is pointing to invalid memory
+     * @param key the value to get
+     */
+    bool getAsData(const std::string&, uint32_t& len, char*& value) const;
+
+    /*!
+     * @param keySet the is set with key to type pairs contained in the ContentValue instance
+     */
+    void getKeyTypeMap(std::map<std::string, uint8_t>& keySet) const;
+
+    /*!
+     * @param key the key of the key value pair to remove
+     * @return true if key was found and removed, false otherwise
+     */
+    bool removeKeyValue(const std::string& key);
+
+    /*!
+     * clears this data structure of all its key value pairs held
+     */
+    void clear();
+
+private:
+
+    /*!
+     * release memory resource associated with mKvData
+     */
+    void clearData();
+
+private:
+
+    std::map<std::string, int32_t> mKvInt32;
+    std::map<std::string, int64_t> mKvInt64;
+    std::map<std::string, double> mKvDouble;
+    std::map<std::string, std::string> mKvString;
+    std::map<std::string, std::pair<uint32_t, char*> > mKvData;
+    std::map<std::string, bool> mKvBool;
+
+    std::map<std::string, uint8_t> mKvSet;
+
+};
+
+
+#endif // RSSQLITE_H

retroshare-gui/src/RetroShare.pro

@@ -1,4 +1,20 @@
-CONFIG += qt gui uic qrc resources uitools idle bitdht# framecatcher# blogs
+CONFIG += qt gui uic qrc resources uitools idle bitdht
+
+# Below is for GXS services.
+#CONFIG += photoshare 
+#CONFIG += wikipoos 
+#CONFIG += thewire 
+#CONFIG += identities
+#CONFIG += forumsv2 
+#CONFIG += posted 
+#CONFIG += unfinished 
+
+
+# Other Disabled Bits.
+#CONFIG += framecatcher
+#CONFIG += blogs
+
+
 QT     += network xml script 
 
 TEMPLATE = app
@@ -12,6 +28,10 @@ MOC_DIR = temp/moc
 #CONFIG += debug
 debug {
 	QMAKE_CFLAGS += -g
+	QMAKE_CXXFLAGS -= -O2
+	QMAKE_CXXFLAGS += -O0
+	QMAKE_CFLAGS -= -O2
+	QMAKE_CFLAGS += -O0
 }
 
 INCLUDEPATH *= retroshare-gui
@@ -765,42 +785,6 @@ DEFINES += BLOGS
 # DEFINES += RS_USE_LINKS
 # }
 
-unfinished {
-
-DEPENDPATH += gui/unfinished \
-
-HEADERS += gui/unfinished/ApplicationWindow.h \
-           gui/unfinished/CalDialog.h \
-           gui/unfinished/ExampleDialog.h \
-           gui/unfinished/GamesDialog.h \
-           gui/unfinished/PhotoDialog.h \
-           gui/unfinished/PhotoShow.h \
-           gui/unfinished/StatisticDialog.h \
-           gui/unfinished/profile/ProfileView.h \
-           gui/unfinished/profile/ProfileEdit.h
-
-FORMS += gui/unfinished/ApplicationWindow.ui \
-         gui/unfinished/CalDialog.ui \
-         gui/unfinished/ExampleDialog.ui \
-         gui/unfinished/GamesDialog.ui \
-         gui/unfinished/PhotoDialog.ui \
-         gui/unfinished/PhotoShow.ui \
-         gui/unfinished/StatisticDialog.ui \
-         gui/unfinished/profile/ProfileView.ui \
-         gui/unfinished/profile/ProfileEdit.ui
-
-SOURCES += gui/unfinished/ApplicationWindow.cpp \
-           gui/unfinished/CalDialog.cpp \
-           gui/unfinished/ExampleDialog.cpp \
-           gui/unfinished/GamesDialog.cpp \
-           gui/unfinished/PhotoDialog.cpp \
-           gui/unfinished/PhotoShow.cpp \
-           gui/unfinished/StatisticDialog.cpp \
-           gui/unfinished/profile/ProfileView.cpp \
-           gui/unfinished/profile/ProfileEdit.cpp
-
-          DEFINES *= UNFINISHED
-}
 
 idle {
 
@@ -821,3 +805,163 @@ LIBS += -lxine
 DEFINES *= CHANNELS_FRAME_CATCHER
 
 }
+
+# BELOW IS GXS New Services.
+
+unfinished {
+
+DEPENDPATH += gui/unfinished \
+
+HEADERS += gui/unfinished/ApplicationWindow.h \
+           gui/unfinished/CalDialog.h \
+           gui/unfinished/ExampleDialog.h \
+           gui/unfinished/GamesDialog.h \
+
+#           gui/unfinished/profile/ProfileView.h \
+#           gui/unfinished/profile/ProfileEdit.h
+#           gui/unfinished/StatisticDialog.h \
+#           gui/unfinished/PhotoDialog.h \
+#           gui/unfinished/PhotoShow.h \
+
+FORMS += gui/unfinished/ApplicationWindow.ui \
+         gui/unfinished/CalDialog.ui \
+         gui/unfinished/ExampleDialog.ui \
+         gui/unfinished/GamesDialog.ui \
+
+#         gui/unfinished/profile/ProfileView.ui \
+#         gui/unfinished/profile/ProfileEdit.ui
+#         gui/unfinished/StatisticDialog.ui \
+#         gui/unfinished/PhotoDialog.ui \
+#         gui/unfinished/PhotoShow.ui \
+
+SOURCES += gui/unfinished/ApplicationWindow.cpp \
+           gui/unfinished/CalDialog.cpp \
+           gui/unfinished/ExampleDialog.cpp \
+           gui/unfinished/GamesDialog.cpp \
+
+#           gui/unfinished/profile/ProfileView.cpp \
+#           gui/unfinished/profile/ProfileEdit.cpp
+#           gui/unfinished/StatisticDialog.cpp \
+#           gui/unfinished/PhotoDialog.cpp \
+#           gui/unfinished/PhotoShow.cpp \
+
+          DEFINES *= UNFINISHED
+}
+
+
+photoshare {
+
+	HEADERS += gui/PhotoShare/PhotoItem.h \
+		gui/PhotoShare/PhotoDialog.h \
+		gui/PhotoShare/PhotoAddDialog.h \
+		gui/PhotoShare/PhotoDetailsDialog.h \
+		gui/PhotoShare/PhotoDrop.h \
+		gui/PhotoShare/PhotoSlideShow.h \
+
+	FORMS += gui/PhotoShare/PhotoItem.ui \
+		gui/PhotoShare/PhotoDialog.ui \
+		gui/PhotoShare/PhotoAddDialog.ui \
+		gui/PhotoShare/PhotoDetailsDialog.ui \
+		gui/PhotoShare/PhotoSlideShow.ui \
+
+	SOURCES += gui/PhotoShare/PhotoItem.cpp \
+		gui/PhotoShare/PhotoDialog.cpp \
+		gui/PhotoShare/PhotoAddDialog.cpp \
+		gui/PhotoShare/PhotoDetailsDialog.cpp \
+		gui/PhotoShare/PhotoDrop.cpp \
+		gui/PhotoShare/PhotoSlideShow.cpp \
+
+}
+
+
+wikipoos {
+
+	HEADERS += gui/WikiPoos/WikiDialog.h \
+		gui/WikiPoos/WikiAddDialog.h \
+		gui/WikiPoos/WikiEditDialog.h \
+
+	FORMS += gui/WikiPoos/WikiDialog.ui \
+		gui/WikiPoos/WikiAddDialog.ui \
+		gui/WikiPoos/WikiEditDialog.ui \
+
+	SOURCES += gui/WikiPoos/WikiDialog.cpp \
+		gui/WikiPoos/WikiAddDialog.cpp \
+		gui/WikiPoos/WikiEditDialog.cpp \
+
+}
+
+
+
+thewire {
+
+	HEADERS += gui/TheWire/PulseItem.h \
+		gui/TheWire/WireDialog.h \
+		gui/TheWire/PulseAddDialog.h \
+
+	FORMS += gui/TheWire/PulseItem.ui \
+		gui/TheWire/WireDialog.ui \
+		gui/TheWire/PulseAddDialog.ui \
+
+	SOURCES += gui/TheWire/PulseItem.cpp \
+		gui/TheWire/WireDialog.cpp \
+		gui/TheWire/PulseAddDialog.cpp \
+
+}
+
+identities {
+
+	HEADERS += util/TokenQueue.h \
+		gui/Identity/IdDialog.h \
+		gui/Identity/IdEditDialog.h \
+
+	FORMS += gui/Identity/IdDialog.ui \
+		gui/Identity/IdEditDialog.ui \
+
+	SOURCES += util/TokenQueue.cpp \
+		gui/Identity/IdDialog.cpp \
+		gui/Identity/IdEditDialog.cpp \
+
+}
+
+
+forumsv2 {
+
+	HEADERS += gui/ForumsV2Dialog.h \
+            gui/forumsv2/ForumV2Details.h \
+            gui/forumsv2/EditForumV2Details.h \
+            gui/forumsv2/CreateForumV2.h \
+            gui/forumsv2/CreateForumV2Msg.h \
+
+	FORMS += gui/ForumsV2Dialog.ui \
+            gui/forumsv2/ForumV2Details.ui \
+            gui/forumsv2/EditForumV2Details.ui \
+            gui/forumsv2/CreateForumV2.ui \
+            gui/forumsv2/CreateForumV2Msg.ui \
+
+	SOURCES += gui/ForumsV2Dialog.cpp \
+            gui/forumsv2/ForumV2Details.cpp \
+            gui/forumsv2/EditForumV2Details.cpp \
+            gui/forumsv2/CreateForumV2.cpp \
+            gui/forumsv2/CreateForumV2Msg.cpp \
+
+}
+
+
+posted {
+
+	HEADERS += gui/Posted/PostedDialog.h \
+            gui/Posted/PostedListDialog.h \
+            gui/Posted/PostedItem.h \
+            gui/Posted/PostedComments.h \
+
+	FORMS += gui/Posted/PostedDialog.ui \
+            gui/Posted/PostedListDialog.ui \
+            gui/Posted/PostedItem.ui \
+            gui/Posted/PostedComments.ui \
+
+	SOURCES += gui/Posted/PostedDialog.cpp \
+            gui/Posted/PostedListDialog.cpp \
+            gui/Posted/PostedItem.cpp \
+            gui/Posted/PostedComments.cpp \
+
+}

retroshare-gui/src/gui/ForumsV2Dialog.h

@@ -0,0 +1,265 @@
+/****************************************************************
+ *  RetroShare is distributed under the following license:
+ *
+ *  Copyright (C) 2008 Robert Fernie
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor, 
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#ifndef _FORUMSV2DIALOG_H
+#define _FORUMSV2DIALOG_H
+
+#include <QThread>
+
+#include "mainpage.h"
+#include "RsAutoUpdatePage.h"
+#include "ui_ForumsV2Dialog.h"
+
+#include <inttypes.h>
+
+#include "util/TokenQueue.h"
+
+#include <retroshare/rsforumsv2.h>
+
+class ForumInfo;
+
+
+/* These are all the parameters that are required for thread loading. 
+ * They are kept static for the load duration.
+ */
+
+class ForumsV2ThreadLoadParameters
+{
+	public:
+		
+		std::string ForumId;
+    		std::string FocusMsgId;
+
+		uint32_t SubscribeFlags;
+		int ViewType;
+		uint32_t FilterColumn;
+	
+		std::map<uint32_t, QTreeWidgetItem *> MsgTokens;
+    		QList<QTreeWidgetItem*> Items;
+    		QList<QTreeWidgetItem*> ItemToExpand;
+		
+		bool FillComplete;
+		bool FlatView;
+		bool UseChildTS;
+		bool ExpandNewMessages;
+};
+
+
+
+
+
+
+class ForumsV2Dialog : public RsAutoUpdatePage, public TokenResponse 
+{
+  Q_OBJECT
+
+public:
+    ForumsV2Dialog(QWidget *parent = 0);
+    ~ForumsV2Dialog();
+
+    bool navigate(const std::string& forumId, const std::string& msgId);
+
+    /* overloaded from RsAuthUpdatePage */
+    virtual void updateDisplay();
+
+protected:
+    bool eventFilter(QObject *obj, QEvent *ev);
+
+private slots:
+    /** Create the context popup menu and it's submenus */
+    void forumListCustomPopupMenu( QPoint point );
+    void threadListCustomPopupMenu( QPoint point );
+    void restoreForumKeys();
+    void newforum();
+
+    void changedForum(const QString &id);
+    void changedThread();
+    void clickedThread (QTreeWidgetItem *item, int column);
+
+    void replytomessage();
+	void replyMessageData(const RsForumV2Msg &msg);
+
+    //void print();
+    //void printpreview();
+
+    //void removemessage();
+    void markMsgAsRead();
+    void markMsgAsReadChildren();
+    void markMsgAsReadAll();
+    void markMsgAsUnread();
+    void markMsgAsUnreadAll();
+    void markMsgAsUnreadChildren();
+    void copyForumLink();
+    void copyMessageLink();
+
+    /* handle splitter */
+    void togglethreadview();
+
+    void createthread();
+    void createmessage();
+
+    void subscribeToForum();
+    void unsubscribeToForum();
+
+    void showForumDetails();
+    void editForumDetails();
+
+    void previousMessage ();
+    void nextMessage ();
+	 void nextUnreadMessage();
+    void downloadAllFiles();
+
+    void changedViewBox();
+
+    void filterColumnChanged();
+    void filterRegExpChanged();
+    void clearFilter();
+
+    void generateMassData();
+
+    void fillThreadFinished();
+    void fillThreadProgress(int current, int count);
+
+    void shareKey();
+
+private:
+    void insertForums();
+    void insertThreads();
+    void insertPost();
+	void insertPostData(const RsForumV2Msg &msg); // Second Half.
+
+    void updateMessageSummaryList(std::string forumId);
+    //void forumInfoToGroupItemInfo(const ForumInfo &forumInfo, GroupItemInfo &groupItemInfo);
+    void forumInfoToGroupItemInfo(const RsGroupMetaData &forumInfo, GroupItemInfo &groupItemInfo);
+
+    void forumSubscribe(bool subscribe);
+    void FillThreads(QList<QTreeWidgetItem *> &ThreadList, bool bExpandNewMessages, QList<QTreeWidgetItem*> &itemToExpand);
+    void FillChildren(QTreeWidgetItem *Parent, QTreeWidgetItem *NewParent, bool bExpandNewMessages, QList<QTreeWidgetItem*> &itemToExpand);
+
+    int getSelectedMsgCount(QList<QTreeWidgetItem*> *pRows, QList<QTreeWidgetItem*> *pRowsRead, QList<QTreeWidgetItem*> *pRowsUnread);
+    void setMsgAsReadUnread(QList<QTreeWidgetItem*> &Rows, bool bRead);
+    void markMsgAsReadUnread(bool bRead, bool bChildren, bool bForum);
+    void CalculateIconsAndFonts(QTreeWidgetItem *pItem = NULL);
+    void CalculateIconsAndFonts(QTreeWidgetItem *pItem, bool &bHasReadChilddren, bool &bHasUnreadChilddren);
+
+    void processSettings(bool bLoad);
+    void togglethreadview_internal();
+
+    void FilterItems();
+    bool FilterItem(QTreeWidgetItem *pItem, QString &sPattern, int filterColumn);
+
+
+	// New Request/Response Loading Functions.
+	void insertForumsData(const std::list<RsGroupMetaData> &forumList);
+	void insertForumThreads(const RsGroupMetaData &fi);
+
+	void requestGroupSummary();
+	void loadGroupSummary(const uint32_t &token);
+
+	void requestGroupSummary_CurrentForum(const std::string &forumId);
+	void loadGroupSummary_CurrentForum(const uint32_t &token);
+
+	void loadCurrentForumThreads(const std::string &forumId);
+	void requestGroupThreadData_InsertThreads(const std::string &forumId);
+	void loadGroupThreadData_InsertThreads(const uint32_t &token);
+	void loadForumBaseThread(const RsForumV2Msg &msg);
+
+	void requestChildData_InsertThreads(uint32_t &token, const std::string &parentId);
+	void loadChildData_InsertThreads(const uint32_t &token);
+	void loadForumChildMsg(const RsForumV2Msg &msg, QTreeWidgetItem *parent);
+
+	void requestMsgData_InsertPost(const std::string &msgId);
+	void loadMsgData_InsertPost(const uint32_t &token);
+	void requestMsgData_ReplyMessage(const std::string &msgId);
+	void loadMsgData_ReplyMessage(const uint32_t &token);
+
+	bool convertMsgToThreadWidget(const RsForumV2Msg &msgInfo, std::string authorName,
+                                        bool useChildTS, uint32_t filterColumn, QTreeWidgetItem *item);
+
+	// Callback for all Loads.
+	void loadRequest(const TokenQueue *queue, const TokenRequest &req);
+
+	TokenQueue *mForumQueue;
+
+
+    bool m_bProcessSettings;
+
+    QTreeWidgetItem *yourForums;
+    QTreeWidgetItem *subscribedForums;
+    QTreeWidgetItem *popularForums;
+    QTreeWidgetItem *otherForums;
+
+    std::string mCurrForumId;
+    std::string mCurrThreadId;
+    int subscribeFlags;
+
+    QFont m_ForumNameFont;
+    int lastViewType;
+    std::string lastForumID;
+
+    //ForumsV2FillThread *fillThread;
+
+    // New Datatypes to replace the FillThread.
+    bool mThreadLoading;
+    ForumsV2ThreadLoadParameters mThreadLoad;
+   
+
+    /** Qt Designer generated object */
+    Ui::ForumsV2Dialog ui;
+};
+
+#if 0
+class ForumsV2FillThread : public QThread
+{
+    Q_OBJECT
+
+public:
+    ForumsV2FillThread(ForumsV2Dialog *parent);
+    ~ForumsV2FillThread();
+
+    void run();
+    void stop();
+    bool wasStopped() { return stopped; }
+
+signals:
+    void progress(int current, int count);
+
+public:
+    std::string forumId;
+    int filterColumn;
+    int subscribeFlags;
+    bool fillComplete;
+    int viewType;
+    bool expandNewMessages;
+    std::string focusMsgId;
+
+    QList<QTreeWidgetItem*> Items;
+    QList<QTreeWidgetItem*> ItemToExpand;
+
+private:
+    volatile bool stopped;
+};
+
+#endif
+
+
+#endif
+

retroshare-gui/src/gui/Identity/IdDialog.cpp

@@ -0,0 +1,382 @@
+/*
+ * Retroshare Identity.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "IdDialog.h"
+
+#include <retroshare/rspeers.h>
+#include <retroshare/rsidentity.h>
+
+#include <iostream>
+#include <sstream>
+
+#include <QTimer>
+
+/******
+ * #define ID_DEBUG 1
+ *****/
+
+// Data Requests.
+#define IDDIALOG_IDLIST		1
+#define IDDIALOG_IDDETAILS	2
+
+/****************************************************************
+ */
+
+
+#define RSID_COL_NICKNAME	0
+#define RSID_COL_KEYID		1
+#define RSID_COL_IDTYPE		2
+
+
+
+#define RSID_REQ_IDLIST		1
+#define RSID_REQ_IDDETAILS	2
+#define RSID_REQ_IDLISTDATA	3
+#define RSID_REQ_IDEDIT		4
+
+/** Constructor */
+IdDialog::IdDialog(QWidget *parent)
+: MainPage(parent)
+{
+	ui.setupUi(this);
+
+	mEditDialog = NULL;
+	//mPulseSelected = NULL;
+
+	ui.radioButton_ListAll->setChecked(true);
+	connect( ui.pushButton_NewId, SIGNAL(clicked()), this, SLOT(OpenOrShowAddDialog()));
+	connect( ui.pushButton_EditId, SIGNAL(clicked()), this, SLOT(OpenOrShowEditDialog()));
+	connect( ui.treeWidget_IdList, SIGNAL(itemSelectionChanged()), this, SLOT(updateSelection()));
+
+	connect( ui.radioButton_ListYourself, SIGNAL(toggled( bool ) ), this, SLOT(ListTypeToggled( bool ) ) );
+	connect( ui.radioButton_ListFriends, SIGNAL(toggled( bool ) ), this, SLOT(ListTypeToggled( bool ) ) );
+	connect( ui.radioButton_ListOthers, SIGNAL(toggled( bool ) ), this, SLOT(ListTypeToggled( bool ) ) );
+	connect( ui.radioButton_ListPseudo, SIGNAL(toggled( bool ) ), this, SLOT(ListTypeToggled( bool ) ) );
+	connect( ui.radioButton_ListAll, SIGNAL(toggled( bool ) ), this, SLOT(ListTypeToggled( bool ) ) );
+
+	QTimer *timer = new QTimer(this);
+	timer->connect(timer, SIGNAL(timeout()), this, SLOT(checkUpdate()));
+	timer->start(1000);
+
+	rsIdentity->generateDummyData();
+
+	mIdQueue = new TokenQueue(rsIdentity, this);
+
+}
+
+void IdDialog::ListTypeToggled(bool checked)
+{
+        if (checked)
+        {
+                requestIdList();
+        }
+}
+
+
+
+void IdDialog::updateSelection()
+{
+	/* */
+	QTreeWidgetItem *item = ui.treeWidget_IdList->currentItem();
+
+	if (!item)
+	{
+		blankSelection();
+	}
+	else
+	{
+		std::string id = item->text(RSID_COL_KEYID).toStdString();
+		requestIdDetails(id);
+	}
+}
+
+
+void IdDialog::blankSelection()
+{
+	/* blank it all - and fix buttons */
+	ui.lineEdit_Nickname->setText("");
+	ui.lineEdit_KeyId->setText("");
+	ui.lineEdit_GpgHash->setText("");
+	ui.lineEdit_GpgId->setText("");
+	ui.lineEdit_GpgName->setText("");
+	ui.lineEdit_GpgEmail->setText("");
+
+	ui.pushButton_Reputation->setEnabled(false);
+	ui.pushButton_Delete->setEnabled(false);
+	ui.pushButton_EditId->setEnabled(false);
+	ui.pushButton_NewId->setEnabled(true);
+}
+
+
+
+
+void IdDialog::requestIdDetails(std::string &id)
+{
+	RsTokReqOptions opts;
+
+	uint32_t token;
+	std::list<std::string> groupIds;
+	groupIds.push_back(id);
+
+        mIdQueue->requestGroupInfo(token, RS_TOKREQ_ANSTYPE_DATA, opts, groupIds, IDDIALOG_IDDETAILS);
+}
+
+
+void IdDialog::insertIdDetails(uint32_t token)
+{
+	/* get details from libretroshare */
+	RsIdGroup data;
+	if (!rsIdentity->getGroupData(token, data))
+	{
+		ui.lineEdit_KeyId->setText("ERROR GETTING KEY!");
+		return;
+	}
+
+	/* get GPG Details from rsPeers */
+	std::string gpgid  = rsPeers->getGPGOwnId();
+	RsPeerDetails details;
+	rsPeers->getPeerDetails(gpgid, details);
+	
+	//ui.lineEdit_Nickname->setText(QString::fromStdString(data.mNickname));
+	ui.lineEdit_Nickname->setText(QString::fromStdString(data.mMeta.mGroupName));
+	//ui.lineEdit_KeyId->setText(QString::fromStdString(data.mKeyId));
+	ui.lineEdit_KeyId->setText(QString::fromStdString(data.mMeta.mGroupId));
+	ui.lineEdit_GpgHash->setText(QString::fromStdString(data.mGpgIdHash));
+	ui.lineEdit_GpgId->setText(QString::fromStdString(data.mGpgId));
+	ui.lineEdit_GpgName->setText(QString::fromStdString(data.mGpgName));
+	ui.lineEdit_GpgEmail->setText(QString::fromStdString(data.mGpgEmail));
+	
+	if (data.mIdType & RSID_RELATION_YOURSELF)
+	{
+		ui.radioButton_IdYourself->setChecked(true);
+	}
+	else if (data.mIdType & RSID_TYPE_PSEUDONYM)
+	{
+		ui.radioButton_IdPseudo->setChecked(true);
+	}
+	else if (data.mIdType & RSID_RELATION_FRIEND)
+	{
+		ui.radioButton_IdFriend->setChecked(true);
+	}
+	else if (data.mIdType & RSID_RELATION_FOF)
+	{
+		ui.radioButton_IdFOF->setChecked(true);
+	}
+	else 
+	{
+		ui.radioButton_IdOther->setChecked(true);
+	}
+	
+	ui.pushButton_NewId->setEnabled(true);
+	if (data.mIdType & RSID_RELATION_YOURSELF)
+	{
+		ui.pushButton_Reputation->setEnabled(false);
+		ui.pushButton_Delete->setEnabled(true);
+		ui.pushButton_EditId->setEnabled(true);
+	}
+	else
+	{
+		ui.pushButton_Reputation->setEnabled(true);
+		ui.pushButton_Delete->setEnabled(false);
+		ui.pushButton_EditId->setEnabled(false);
+	}
+}
+
+void IdDialog::checkUpdate()
+{
+	/* update */
+	if (!rsIdentity)
+		return;
+
+	if (rsIdentity->updated())
+	{
+		requestIdList();
+	}
+	return;
+}
+
+
+void IdDialog::OpenOrShowAddDialog()
+{
+	if (!mEditDialog)
+	{
+		mEditDialog = new IdEditDialog(NULL);
+	}
+	bool pseudo = false;
+	mEditDialog->setupNewId(pseudo);
+
+	mEditDialog->show();
+
+}
+
+
+void IdDialog::OpenOrShowEditDialog()
+{
+	if (!mEditDialog)
+	{
+		mEditDialog = new IdEditDialog(NULL);
+	}
+
+
+	/* */
+	QTreeWidgetItem *item = ui.treeWidget_IdList->currentItem();
+
+	if (!item)
+	{
+		std::cerr << "IdDialog::OpenOrShowEditDialog() Invalid item";
+		std::cerr << std::endl;
+		return;
+	}
+
+	std::string keyId = item->text(RSID_COL_KEYID).toStdString();
+
+	if (mEditDialog)
+	{
+		mEditDialog->setupExistingId(keyId);
+		mEditDialog->show();
+	}
+}
+
+void IdDialog::requestIdList()
+{
+	RsTokReqOptions opts;
+
+	uint32_t token;
+	std::list<std::string> groupIds;
+
+        mIdQueue->requestGroupInfo(token, RS_TOKREQ_ANSTYPE_DATA, opts, groupIds, IDDIALOG_IDLIST);
+}
+
+
+void IdDialog::insertIdList(uint32_t token)
+{
+	QTreeWidget *tree = ui.treeWidget_IdList;
+
+	tree->clear();
+
+	std::list<std::string> ids;
+	std::list<std::string>::iterator it;
+
+	bool acceptAll = ui.radioButton_ListAll->isChecked();
+	bool acceptPseudo = ui.radioButton_ListPseudo->isChecked();
+	bool acceptYourself = ui.radioButton_ListYourself->isChecked();
+	bool acceptFriends = ui.radioButton_ListFriends->isChecked();
+	bool acceptOthers = ui.radioButton_ListOthers->isChecked();
+
+	//rsIdentity->getIdentityList(ids);
+
+	//for(it = ids.begin(); it != ids.end(); it++)
+	//{
+	RsIdGroup data;
+	while(rsIdentity->getGroupData(token, data))
+	{
+
+		/* do filtering */
+		bool ok = false;
+		if (acceptAll)
+		{
+			ok = true;
+		}
+		else if (data.mIdType & RSID_TYPE_PSEUDONYM)
+		{
+ 			if (acceptPseudo)
+			{
+				ok = true;
+			}
+
+			if ((data.mIdType & RSID_RELATION_YOURSELF) && (acceptYourself))
+			{
+				ok = true;
+			}
+		}
+		else
+		{
+			if (data.mIdType & RSID_RELATION_YOURSELF)
+			{
+ 				if (acceptYourself)
+				{
+					ok = true;
+				}
+			}
+			else if (data.mIdType & (RSID_RELATION_FRIEND | RSID_RELATION_FOF)) 
+			{
+				if (acceptFriends)
+				{
+					ok = true;
+				}
+			}
+			else 
+			{
+				if (acceptOthers)
+				{
+					ok = true;
+				}
+			}
+		}
+
+		if (!ok)
+		{
+			continue;
+		}
+
+
+		QTreeWidgetItem *item = new QTreeWidgetItem(NULL);
+		//item->setText(RSID_COL_NICKNAME, QString::fromStdString(data.mNickname));
+		//item->setText(RSID_COL_KEYID, QString::fromStdString(data.mKeyId));
+		item->setText(RSID_COL_NICKNAME, QString::fromStdString(data.mMeta.mGroupName));
+		item->setText(RSID_COL_KEYID, QString::fromStdString(data.mMeta.mGroupId));
+		item->setText(RSID_COL_IDTYPE, QString::fromStdString(rsIdTypeToString(data.mIdType)));
+
+                tree->addTopLevelItem(item);
+	}
+
+	// fix up buttons.
+	updateSelection();
+}
+
+void IdDialog::loadRequest(const TokenQueue *queue, const TokenRequest &req)
+{
+        std::cerr << "IdDialog::loadRequest() UserType: " << req.mUserType;
+        std::cerr << std::endl;
+		
+	switch(req.mUserType)
+	{
+		case IDDIALOG_IDLIST:
+			insertIdList(req.mToken);
+			break;
+		
+		case IDDIALOG_IDDETAILS:
+			insertIdDetails(req.mToken);
+			break;
+		
+		default:
+        		std::cerr << "IdDialog::loadRequest() ERROR";
+        		std::cerr << std::endl;
+			break;
+
+	}
+}
+
+		
+
+

retroshare-gui/src/gui/Identity/IdDialog.h

@@ -0,0 +1,79 @@
+/*
+ * Retroshare Identity.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef MRK_IDENTITY_DIALOG_H
+#define MRK_IDENTITY_DIALOG_H
+
+#include "gui/mainpage.h"
+#include "ui_IdDialog.h"
+
+#include <retroshare/rsidentity.h>
+
+#include <map>
+
+#include "gui/Identity/IdEditDialog.h"
+#include "util/TokenQueue.h"
+
+class IdDialog : public MainPage, public TokenResponse
+{
+  Q_OBJECT
+
+public:
+	IdDialog(QWidget *parent = 0);
+
+void    loadRequest(const TokenQueue *queue, const TokenRequest &req);
+
+private slots:
+
+
+	void ListTypeToggled(bool checked);
+	void checkUpdate();
+	void OpenOrShowAddDialog();
+	void OpenOrShowEditDialog();
+
+	void updateSelection();
+
+private:
+
+	void blankSelection();
+	void requestIdDetails(std::string &id);
+	void insertIdDetails(uint32_t token);
+
+	void requestIdList();
+	void requestIdData(std::list<std::string> &ids);
+	void insertIdList(uint32_t token);
+
+	void requestIdEdit(std::string &id);
+	void showIdEdit(uint32_t token);
+
+	IdEditDialog *mEditDialog;
+
+	TokenQueue *mIdQueue;
+
+	/* UI - from Designer */
+	Ui::IdDialog ui;
+
+};
+
+#endif
+

retroshare-gui/src/gui/Identity/IdDialog.ui

@@ -0,0 +1,365 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <class>IdDialog</class>
+ <widget class="QWidget" name="IdDialog">
+  <property name="geometry">
+   <rect>
+    <x>0</x>
+    <y>0</y>
+    <width>744</width>
+    <height>647</height>
+   </rect>
+  </property>
+  <property name="windowTitle">
+   <string/>
+  </property>
+  <layout class="QGridLayout" name="gridLayout_3">
+   <item row="0" column="0" rowspan="3">
+    <layout class="QVBoxLayout" name="verticalLayout_2">
+     <item>
+      <widget class="QGroupBox" name="groupBox">
+       <property name="title">
+        <string>Showing:</string>
+       </property>
+       <layout class="QVBoxLayout" name="verticalLayout">
+        <item>
+         <widget class="QRadioButton" name="radioButton_ListYourself">
+          <property name="text">
+           <string>Yourself</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_ListFriends">
+          <property name="text">
+           <string>Friends / Friends of Friends</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_ListOthers">
+          <property name="text">
+           <string>Others</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="Line" name="line_3">
+          <property name="orientation">
+           <enum>Qt::Horizontal</enum>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_ListPseudo">
+          <property name="text">
+           <string>Pseudonyms</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="Line" name="line_4">
+          <property name="orientation">
+           <enum>Qt::Horizontal</enum>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_ListAll">
+          <property name="text">
+           <string>All</string>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <layout class="QHBoxLayout" name="horizontalLayout"/>
+        </item>
+        <item>
+         <widget class="QLineEdit" name="lineEdit">
+          <property name="text">
+           <string>filter</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </widget>
+     </item>
+     <item>
+      <widget class="QTreeWidget" name="treeWidget_IdList">
+       <column>
+        <property name="text">
+         <string>Nickname</string>
+        </property>
+       </column>
+       <column>
+        <property name="text">
+         <string>KeyId</string>
+        </property>
+       </column>
+       <column>
+        <property name="text">
+         <string>Type</string>
+        </property>
+       </column>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="0" column="1">
+    <layout class="QHBoxLayout" name="horizontalLayout_2">
+     <item>
+      <widget class="QPushButton" name="pushButton_Delete">
+       <property name="text">
+        <string>Delete ID</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="pushButton_Reputation">
+       <property name="text">
+        <string>Edit Reputation</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="pushButton_EditId">
+       <property name="text">
+        <string>Edit ID</string>
+       </property>
+      </widget>
+     </item>
+     <item>
+      <widget class="QPushButton" name="pushButton_NewId">
+       <property name="text">
+        <string>New ID</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item row="1" column="1">
+    <widget class="QGroupBox" name="groupBox_2">
+     <property name="title">
+      <string>Identity Type</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout">
+      <item row="0" column="0">
+       <widget class="QLabel" name="label">
+        <property name="text">
+         <string>Nickname</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="QLineEdit" name="lineEdit_Nickname">
+        <property name="enabled">
+         <bool>true</bool>
+        </property>
+        <property name="readOnly">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="2" rowspan="6">
+       <layout class="QVBoxLayout" name="verticalLayout_3">
+        <item>
+         <widget class="QRadioButton" name="radioButton_IdYourself">
+          <property name="enabled">
+           <bool>false</bool>
+          </property>
+          <property name="text">
+           <string>Yourself</string>
+          </property>
+          <property name="checkable">
+           <bool>true</bool>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_IdFriend">
+          <property name="enabled">
+           <bool>false</bool>
+          </property>
+          <property name="text">
+           <string>Friend</string>
+          </property>
+          <property name="checkable">
+           <bool>true</bool>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_IdFOF">
+          <property name="enabled">
+           <bool>false</bool>
+          </property>
+          <property name="text">
+           <string>Friend of Friend</string>
+          </property>
+          <property name="checkable">
+           <bool>true</bool>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_IdOther">
+          <property name="enabled">
+           <bool>false</bool>
+          </property>
+          <property name="text">
+           <string>Other</string>
+          </property>
+          <property name="checkable">
+           <bool>true</bool>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="Line" name="line_2">
+          <property name="orientation">
+           <enum>Qt::Horizontal</enum>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QRadioButton" name="radioButton_IdPseudo">
+          <property name="enabled">
+           <bool>false</bool>
+          </property>
+          <property name="text">
+           <string>Pseudonym</string>
+          </property>
+          <property name="checkable">
+           <bool>true</bool>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="label_4">
+        <property name="text">
+         <string>Key ID</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <widget class="QLineEdit" name="lineEdit_KeyId">
+        <property name="enabled">
+         <bool>true</bool>
+        </property>
+        <property name="readOnly">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="4" column="0">
+       <widget class="QLabel" name="label_3">
+        <property name="text">
+         <string>GPG Name</string>
+        </property>
+       </widget>
+      </item>
+      <item row="4" column="1">
+       <widget class="QLineEdit" name="lineEdit_GpgName">
+        <property name="enabled">
+         <bool>true</bool>
+        </property>
+        <property name="readOnly">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="5" column="0">
+       <widget class="QLabel" name="label_5">
+        <property name="text">
+         <string>GPG Email</string>
+        </property>
+       </widget>
+      </item>
+      <item row="5" column="1">
+       <widget class="QLineEdit" name="lineEdit_GpgEmail">
+        <property name="enabled">
+         <bool>true</bool>
+        </property>
+        <property name="readOnly">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <widget class="QLineEdit" name="lineEdit_GpgHash">
+        <property name="enabled">
+         <bool>true</bool>
+        </property>
+        <property name="readOnly">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="3" column="1">
+       <widget class="QLineEdit" name="lineEdit_GpgId">
+        <property name="enabled">
+         <bool>true</bool>
+        </property>
+        <property name="readOnly">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item row="3" column="0">
+       <widget class="QLabel" name="label_7">
+        <property name="text">
+         <string>GPG Id</string>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="label_8">
+        <property name="text">
+         <string>GPG Hash</string>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item row="2" column="1">
+    <widget class="QGroupBox" name="groupBox_3">
+     <property name="title">
+      <string>Reputation</string>
+     </property>
+     <layout class="QGridLayout" name="gridLayout_2">
+      <item row="0" column="0">
+       <widget class="QLabel" name="label_6">
+        <property name="text">
+         <string>Your Rating</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <widget class="QSpinBox" name="spinBox"/>
+      </item>
+      <item row="0" column="2">
+       <widget class="QLabel" name="label_2">
+        <property name="text">
+         <string>Overall Rating</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="3">
+       <widget class="QLineEdit" name="lineEdit_3"/>
+      </item>
+      <item row="1" column="0" colspan="4">
+       <widget class="QListWidget" name="listWidget"/>
+      </item>
+     </layout>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources>
+  <include location="../LinksCloud/images.qrc"/>
+ </resources>
+ <connections/>
+</ui>

retroshare-gui/src/gui/Identity/IdEditDialog.cpp

@@ -0,0 +1,256 @@
+/*
+ * Retroshare Identity
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "gui/Identity/IdEditDialog.h"
+#include "util/TokenQueue.h"
+
+#include <retroshare/rsidentity.h>
+#include <retroshare/rspeers.h>
+
+#include <iostream>
+
+#define IDEDITDIALOG_LOADID	1
+
+/** Constructor */
+IdEditDialog::IdEditDialog(QWidget *parent)
+: QWidget(parent)
+{
+	ui.setupUi(this);
+
+	connect(ui.radioButton_GpgId, SIGNAL( toggled( bool ) ), this, SLOT( IdTypeToggled( bool ) ) );
+	connect(ui.radioButton_Pseudo, SIGNAL( toggled( bool ) ), this, SLOT( IdTypeToggled( bool ) ) );
+	connect(ui.pushButton_Update, SIGNAL( clicked( void ) ), this, SLOT( updateId( void ) ) );
+	connect(ui.pushButton_Cancel, SIGNAL( clicked( void ) ), this, SLOT( cancelId( void ) ) );
+
+	mIdQueue = new TokenQueue(rsIdentity, this);
+}
+
+void IdEditDialog::setupNewId(bool pseudo)
+{
+	ui.checkBox_NewId->setChecked(true);
+	ui.checkBox_NewId->setEnabled(false);
+	ui.lineEdit_KeyId->setText("To Be Generated");
+	ui.lineEdit_Nickname->setText("");
+	ui.radioButton_GpgId->setEnabled(true);
+	ui.radioButton_Pseudo->setEnabled(true);
+
+	if (pseudo)
+	{
+		ui.radioButton_Pseudo->setChecked(true);
+	}
+	else
+	{
+		ui.radioButton_GpgId->setChecked(true);
+	}
+
+	// force - incase it wasn't triggered.
+	IdTypeToggled(true);
+	return;
+}
+
+void IdEditDialog::IdTypeToggled(bool checked)
+{
+	if (checked)
+	{
+		bool pseudo = ui.radioButton_Pseudo->isChecked();
+		updateIdType(pseudo);
+	}
+}
+
+void IdEditDialog::updateIdType(bool pseudo)
+{
+	if (pseudo)
+	{
+		ui.lineEdit_GpgHash->setText("N/A");
+		ui.lineEdit_GpgId->setText("N/A");
+		ui.lineEdit_GpgName->setText("N/A");
+		ui.lineEdit_GpgEmail->setText("N/A");
+	}
+	else
+	{
+		/* get GPG Details from rsPeers */
+		std::string gpgid  = rsPeers->getGPGOwnId();
+		RsPeerDetails details;
+		rsPeers->getPeerDetails(gpgid, details);
+
+		ui.lineEdit_GpgId->setText(QString::fromStdString(gpgid));
+		ui.lineEdit_GpgHash->setText("To Be Generated");
+		ui.lineEdit_GpgName->setText(QString::fromStdString(details.name));
+		ui.lineEdit_GpgEmail->setText(QString::fromStdString(details.email));
+	}
+	return;
+}
+
+
+void IdEditDialog::setupExistingId(std::string keyId)
+{
+        RsTokReqOptions opts;
+
+        std::list<std::string> groupIds;
+        groupIds.push_back(keyId);
+
+        uint32_t token;
+        mIdQueue->requestGroupInfo(token, RS_TOKREQ_ANSTYPE_DATA, opts, groupIds, IDEDITDIALOG_LOADID);
+}
+
+
+void IdEditDialog::loadExistingId(uint32_t token)
+{
+	ui.checkBox_NewId->setChecked(false);
+	ui.checkBox_NewId->setEnabled(false);
+	ui.radioButton_GpgId->setEnabled(false);
+	ui.radioButton_Pseudo->setEnabled(false);
+
+        /* get details from libretroshare */
+        RsIdGroup data;
+        if (!rsIdentity->getGroupData(token, data))
+        {
+		ui.lineEdit_KeyId->setText("ERROR KEYID INVALID");
+		ui.lineEdit_Nickname->setText("");
+
+		ui.lineEdit_GpgHash->setText("N/A");
+		ui.lineEdit_GpgId->setText("N/A");
+		ui.lineEdit_GpgName->setText("N/A");
+		ui.lineEdit_GpgEmail->setText("N/A");
+		return;
+	}
+	bool pseudo = (data.mIdType & RSID_TYPE_PSEUDONYM);
+
+	if (pseudo)
+	{
+		ui.radioButton_Pseudo->setChecked(true);
+	}
+	else
+	{
+		ui.radioButton_GpgId->setChecked(true);
+	}
+
+	// DOES THIS TRIGGER ALREADY???
+	// force - incase it wasn't triggered.
+	IdTypeToggled(true);
+
+	//ui.lineEdit_Nickname->setText(QString::fromStdString(data.mNickname));
+        //ui.lineEdit_KeyId->setText(QString::fromStdString(data.mKeyId));
+	ui.lineEdit_Nickname->setText(QString::fromStdString(data.mMeta.mGroupName));
+        ui.lineEdit_KeyId->setText(QString::fromStdString(data.mMeta.mGroupId));
+
+	if (pseudo)
+	{
+		ui.lineEdit_GpgHash->setText("N/A");
+		ui.lineEdit_GpgId->setText("N/A");
+		ui.lineEdit_GpgName->setText("N/A");
+		ui.lineEdit_GpgEmail->setText("N/A");
+	}
+	else
+	{
+		ui.lineEdit_GpgHash->setText(QString::fromStdString(data.mGpgIdHash));
+
+		if (data.mGpgIdKnown)
+		{
+			ui.lineEdit_GpgId->setText(QString::fromStdString(data.mGpgId));
+			ui.lineEdit_GpgName->setText(QString::fromStdString(data.mGpgName));
+			ui.lineEdit_GpgEmail->setText(QString::fromStdString(data.mGpgEmail));
+		}
+		else
+		{
+			ui.lineEdit_GpgId->setText("EXIST Unknown");
+			ui.lineEdit_GpgName->setText("Unknown");
+			ui.lineEdit_GpgEmail->setText("Unknown");
+		}
+	}
+
+	return;
+}
+
+void IdEditDialog::updateId()
+{
+	RsIdGroup rid;
+	// Must set, Nickname, KeyId(if existing), mIdType, GpgId.
+
+	rid.mMeta.mGroupName = ui.lineEdit_Nickname->text().toStdString();
+
+	if (rid.mMeta.mGroupName.size() < 2)
+	{
+		std::cerr << "IdEditDialog::updateId() Nickname too short";
+		std::cerr << std::endl;
+		return;
+	}
+
+	rid.mIdType = RSID_RELATION_YOURSELF;
+	if (ui.checkBox_NewId->isChecked())
+	{
+		rid.mMeta.mGroupId = "";
+	}
+	else
+	{
+		rid.mMeta.mGroupId = ui.lineEdit_KeyId->text().toStdString();
+	}
+
+	if (ui.radioButton_GpgId->isChecked())
+	{
+		rid.mIdType |= RSID_TYPE_REALID;
+
+		rid.mGpgId = ui.lineEdit_GpgId->text().toStdString();
+		rid.mGpgIdHash = ui.lineEdit_GpgHash->text().toStdString();
+		rid.mGpgName = ui.lineEdit_GpgName->text().toStdString();
+		rid.mGpgEmail = ui.lineEdit_GpgEmail->text().toStdString();
+	}
+	else
+	{
+		rid.mIdType |= RSID_TYPE_PSEUDONYM;
+
+		rid.mGpgId = "";
+		rid.mGpgIdHash = "";
+		rid.mGpgName = "";
+		rid.mGpgEmail = "";
+	}
+
+	// TODO.
+	//rsIdentity->updateIdentity(rid);
+
+	hide();
+	return;
+}
+
+void IdEditDialog::cancelId()
+{
+	hide();
+	return;
+}
+
+void IdEditDialog::clearDialog()
+{
+	return;
+}
+
+
+void IdEditDialog::loadRequest(const TokenQueue *queue, const TokenRequest &req)
+{
+	std::cerr << "IdDialog::loadRequest() UserType: " << req.mUserType;
+	std::cerr << std::endl;
+	
+	// only one here!
+	loadExistingId(req.mToken);
+}
+
+	

retroshare-gui/src/gui/Identity/IdEditDialog.h

@@ -0,0 +1,64 @@
+/*
+ * Retroshare Identity.
+ *
+ * Copyright 2012-2012 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2.1 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef MRK_ID_EDIT_DIALOG_H
+#define MRK_ID_EDIT_DIALOG_H
+
+#include "ui_IdEditDialog.h"
+
+#include <inttypes.h>
+
+#include "util/TokenQueue.h"
+
+class IdEditDialog : public QWidget, public TokenResponse
+{
+  Q_OBJECT
+
+public:
+	IdEditDialog(QWidget *parent = 0);
+
+	void setupNewId(bool pseudo);
+	void setupExistingId(std::string keyId);
+	void loadExistingId(uint32_t token);
+
+void    loadRequest(const TokenQueue *queue, const TokenRequest &req);
+
+private slots:
+
+	void IdTypeToggled(bool checked);
+	void updateId();
+	void cancelId();
+
+private:
+	void updateIdType(bool pseudo);
+	void clearDialog();
+
+protected:
+
+	Ui::IdEditDialog ui;
+
+	TokenQueue *mIdQueue;
+};
+
+#endif
+