handled validation/sign of message when key not available

(set for 5 attempts before message is dropped)
moved rstokenservice.h to retroshare interface folder
groups do not sync anymore unless user is subscribed to it

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@6194 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/gxs/rsgenexchange.cc

@@ -37,6 +37,7 @@
 #include "util/contentvalue.h"
 #include "retroshare/rsgxsflags.h"
 #include "rsgixs.h"
+#include "rsgxsutil.h"
 
 
 #define PUB_GRP_MASK 0x000f
@@ -56,7 +57,10 @@
 RsGenExchange::RsGenExchange(RsGeneralDataService *gds, RsNetworkExchangeService *ns,
                              RsSerialType *serviceSerialiser, uint16_t servType, RsGixs* gixs, uint32_t authenPolicy)
 : mGenMtx("GenExchange"), mDataStore(gds), mNetService(ns), mSerialiser(serviceSerialiser),
-    mServType(servType), mGixs(gixs), mAuthenPolicy(authenPolicy)
+    mServType(servType), mGixs(gixs), mAuthenPolicy(authenPolicy),
+    CREATE_FAIL(0), CREATE_SUCCESS(1), CREATE_FAIL_TRY_LATER(2), SIGN_MAX_ATTEMPTS(5),
+    SIGN_FAIL(0), SIGN_SUCCESS(1), SIGN_FAIL_TRY_LATER(2),
+    VALIDATE_FAIL(0), VALIDATE_SUCCESS(1), VALIDATE_FAIL_TRY_LATER(2), VALIDATE_MAX_ATTEMPTS(5)
 {
 
     mDataAccess = new RsGxsDataAccess(gds);
@@ -65,7 +69,7 @@ RsGenExchange::RsGenExchange(RsGeneralDataService *gds, RsNetworkExchangeService
 
 RsGenExchange::~RsGenExchange()
 {
-    // need to destruct in a certain order (prob a bad thing!)
+    // need to destruct in a certain order (bad thing, TODO: put down instance ownership rules!)
     delete mNetService;
 
     delete mDataAccess;
@@ -299,14 +303,14 @@ bool RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& privateKeySe
     return ok;
 }
 
-bool RsGenExchange::createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBinaryData& msgData,
+int RsGenExchange::createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBinaryData& msgData,
                                         const RsGxsMsgMetaData& msgMeta, RsGxsGrpMetaData& grpMeta)
 {
-    bool isParent = false;
     bool needPublishSign = false, needIdentitySign = false;
-    bool ok = true;
     uint32_t grpFlag = grpMeta.mGroupFlags;
 
+    bool publishSignSuccess = false;
+
     std::cerr << "RsGenExchange::createMsgSignatures() for Msg.mMsgName: " << msgMeta.mMsgName;
     std::cerr << std::endl;
 
@@ -362,7 +366,6 @@ bool RsGenExchange::createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBina
         std::cerr << std::endl;
     }
 
-
     if(needPublishSign)
     {
         // public and shared is publish key
@@ -380,96 +383,93 @@ bool RsGenExchange::createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBina
                         break;
         }
 
-        if (!pub_key_found)
+        if (pub_key_found)
         {
-            std::cerr << "RsGenExchange::createMsgSignatures()";
+            // private publish key
-            std::cerr << " ERROR Cannot find PUBLISH KEY for Message Signing";
+            pubKey = &(mit->second);
-            std::cerr << std::endl;
+
-            return false;
+            RsTlvKeySignature pubSign = signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_PUBLISH];
+
+            publishSignSuccess = GxsSecurity::getSignature((char*)msgData.bin_data, msgData.bin_len, pubKey, pubSign);
+
+            //place signature in msg meta
+            signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_PUBLISH] = pubSign;
+        }else
+        {
+        	std::cerr << "RsGenExchange::createMsgSignatures()";
+			std::cerr << " ERROR Cannot find PUBLISH KEY for Message Signing!";
+			std::cerr << " ERROR Publish Sign failed!";
+			std::cerr << std::endl;
         }
-        // private publish key
-        pubKey = &(mit->second);
 
-        RsTlvKeySignature pubSign = signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_PUBLISH];
+    }
-
+    else // publish sign not needed so set as successful
-        ok &= GxsSecurity::getSignature((char*)msgData.bin_data, msgData.bin_len, pubKey, pubSign);
+    {
-
+    	publishSignSuccess = true;
-        //place signature in msg meta
-        signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_PUBLISH] = pubSign;
     }
 
+    int id_ret;
 
     if (needIdentitySign)
     {
         if(mGixs)
         {
-		/***************************************************************
-		 * NOTE: The logic below is wrong.
-		 * mGixs->havePrivateKey(msgMeta.mAuthorId) can return False if the Key isn't cached.
-		 *
-		 * This Operation should be retried again - later.
-		 *
-		 **************************************************************/ 
-
             bool haveKey = mGixs->havePrivateKey(msgMeta.mAuthorId);
 
             if(haveKey)
             {
-                mGixs->requestPrivateKey(msgMeta.mAuthorId);
-
                 RsTlvSecurityKey authorKey;
-
+                mGixs->getPrivateKey(msgMeta.mAuthorId, authorKey);
-                double timeDelta = 0.002; // fast polling
-                time_t now = time(NULL);
-
-                bool auth_key_fetched = false;
-                // poll immediately but, don't spend more than a second polling
-                while((!auth_key_fetched) && ((now + 5) > time(NULL)))
-		{
-			auth_key_fetched = (mGixs->getPrivateKey(msgMeta.mAuthorId, authorKey) == 1);
-#ifndef WINDOWS_SYS
-                        usleep((int) (timeDelta * 1000000));
-#else
-                        Sleep((int) (timeDelta * 1000));
-#endif
-                }
-
-
-		if (!auth_key_fetched)
-		{
-                     std::cerr << "RsGenExchange::createMsgSignatures()";
-                     std::cerr << " ERROR Cannot find AUTHOR KEY for Message Signing";
-                     std::cerr << std::endl;
-                     return false;
-                }
-
                 RsTlvKeySignature sign;
-                ok &= GxsSecurity::getSignature((char*)msgData.bin_data, msgData.bin_len,
-                                                &authorKey, sign);
-                signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_IDENTITY] = sign;
 
+                if(GxsSecurity::getSignature((char*)msgData.bin_data, msgData.bin_len,
+                                                &authorKey, sign))
+                {
+                	id_ret = SIGN_SUCCESS;
+                }
+                else
+                {
+                	id_ret = SIGN_FAIL;
+                }
+
+                signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_IDENTITY] = sign;
             }
             else
             {
-                std::cerr << "RsGenExchange::createMsgSignatures()";
+            	mGixs->requestPrivateKey(msgMeta.mAuthorId);
-                std::cerr << " ERROR AUTHOR KEY is not Cached / available for Message Signing";
+
+                std::cerr << "RsGenExchange::createMsgSignatures(): ";
+                std::cerr << " ERROR AUTHOR KEY: " <<  msgMeta.mAuthorId
+                		  << " is not Cached / available for Message Signing\n";
+                std::cerr << "RsGenExchange::createMsgSignatures():  Requestiong AUTHOR KEY";
                 std::cerr << std::endl;
 
-                ok = false;
+                id_ret = SIGN_FAIL_TRY_LATER;
             }
         }
         else
         {
             std::cerr << "RsGenExchange::createMsgSignatures()";
             std::cerr << "Gixs not enabled while request identity signature validation!" << std::endl;
-
+            id_ret = SIGN_FAIL;
-            ok = false;
         }
     }
-    return ok;
+    else
+    {
+    	id_ret = SIGN_SUCCESS;
+    }
+
+    if(publishSignSuccess)
+    {
+    	return id_ret;
+    }
+    else
+    {
+    	return SIGN_FAIL;
+    }
 }
 
-bool RsGenExchange::createMessage(RsNxsMsg* msg)
+int RsGenExchange::createMessage(RsNxsMsg* msg)
 {
 	const RsGxsGroupId& id = msg->grpId;
 
@@ -477,17 +477,20 @@ bool RsGenExchange::createMessage(RsNxsMsg* msg)
 
 	metaMap.insert(std::make_pair(id, (RsGxsGrpMetaData*)(NULL)));
 	mDataStore->retrieveGxsGrpMetaData(metaMap);
-        bool ok = true;
+
-        RsGxsMsgMetaData &meta = *(msg->metaData);
+	RsGxsMsgMetaData &meta = *(msg->metaData);
+
+	int ret_val;
 
 	if(!metaMap[id])
 	{
-            return false;
+		return CREATE_FAIL;
 	}
 	else
 	{
             // get publish key
-            RsGxsGrpMetaData* grpMeta = metaMap[id];
+
+		RsGxsGrpMetaData* grpMeta = metaMap[id];
 
             uint32_t metaDataLen = meta.serial_size();
             uint32_t allMsgDataLen = metaDataLen + msg->msg.bin_len;
@@ -505,7 +508,8 @@ bool RsGenExchange::createMessage(RsNxsMsg* msg)
             msgData.setBinData(allMsgData, allMsgDataLen);
 
             // create signatures
-            ok &= createMsgSignatures(meta.signSet, msgData, meta, *grpMeta);
+           ret_val = createMsgSignatures(meta.signSet, msgData, meta, *grpMeta);
+
 
             // get hash of msg data to create msg id
             pqihash hash;
@@ -521,14 +525,24 @@ bool RsGenExchange::createMessage(RsNxsMsg* msg)
             delete grpMeta;
 	}
 
-	return ok;
+	if(ret_val == SIGN_FAIL)
+		return CREATE_FAIL;
+	else if(ret_val == SIGN_FAIL_TRY_LATER)
+		return CREATE_FAIL_TRY_LATER;
+	else if(ret_val == SIGN_SUCCESS)
+		return CREATE_SUCCESS;
+	else
+	{
+		std::cerr << "Unknown return value from signature attempt!";
+		return CREATE_FAIL;
+	}
 }
 
-bool RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, RsTlvSecurityKeySet& grpKeySet)
+int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, RsTlvSecurityKeySet& grpKeySet)
 {
     bool needIdentitySign = false;
     bool needPublishSign = false;
-    bool valid = true;
+    bool publishValidate = true, idValidate = true;
 
     uint8_t author_flag = GXS_SERV::MSG_AUTHEN_ROOT_AUTHOR_SIGN;
     uint8_t publish_flag = GXS_SERV::MSG_AUTHEN_ROOT_PUBLISH_SIGN;
@@ -582,13 +596,17 @@ bool RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, RsTlvSec
         if(!keyId.empty())
         {
             RsTlvSecurityKey& key = keys[keyId];
-            valid &= GxsSecurity::validateNxsMsg(*msg, sign, key);
+            publishValidate &= GxsSecurity::validateNxsMsg(*msg, sign, key);
         }
         else
         {
-            valid = false;
+            publishValidate = false;
         }
     }
+    else
+    {
+    	publishValidate = true;
+    }
 
 
 
@@ -602,39 +620,27 @@ bool RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, RsTlvSec
             {
 
                 RsTlvSecurityKey authorKey;
+                bool auth_key_fetched = mGixs->getKey(metaData.mAuthorId, authorKey) == 1;
 
-                double timeDelta = 0.002; // fast polling
+				if (auth_key_fetched)
-                time_t now = time(NULL);
+				{
-                bool auth_key_fetched = false;
-                // poll immediately but, don't spend more than a second polling
-                while((!auth_key_fetched) && ((now + 1) > time(NULL)))
-		{
-			auth_key_fetched = (mGixs->getKey(metaData.mAuthorId, authorKey) == 1);
-#ifndef WINDOWS_SYS
-                        usleep((int) (timeDelta * 1000000));
-#else
-                        Sleep((int) (timeDelta * 1000));
-#endif
-                }
 
-
+	                RsTlvKeySignature sign = metaData.signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_IDENTITY];
-		if (!auth_key_fetched)
+	                idValidate &= GxsSecurity::validateNxsMsg(*msg, sign, authorKey);
-		{
+				}
+				else
+				{
                      std::cerr << "RsGenExchange::validateMsg()";
-                     std::cerr << " ERROR Cannot find AUTHOR KEY for Message Validation";
+                     std::cerr << " ERROR Cannot Retrieve AUTHOR KEY for Message Validation";
                      std::cerr << std::endl;
-                     return false;
+                     idValidate = false;
                 }
 
-
-                RsTlvKeySignature sign = metaData.signSet.keySignSet[GXS_SERV::FLAG_AUTHEN_IDENTITY];
-                valid &= GxsSecurity::validateNxsMsg(*msg, sign, authorKey);
-
             }else
             {
                 std::list<std::string> peers;
                 mGixs->requestKey(metaData.mAuthorId, peers);
-                valid = false;
+                return VALIDATE_FAIL_TRY_LATER;
             }
         }
         else
@@ -642,12 +648,19 @@ bool RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, RsTlvSec
 #ifdef GEN_EXHANGE_DEBUG
             std::cerr << "Gixs not enabled while request identity signature validation!" << std::endl;
 #endif
-            valid = false;
+            idValidate = false;
         }
-
+    }
+    else
+    {
+    	idValidate = true;
     }
 
-    return valid;
+    if(publishValidate && idValidate)
+    	return VALIDATE_SUCCESS;
+    else
+    	return VALIDATE_FAIL;
+
 }
 
 bool RsGenExchange::checkMsgAuthenFlag(const PrivacyBitPos& pos, const uint8_t& flag) const
@@ -1070,7 +1083,18 @@ void RsGenExchange::notifyNewMessages(std::vector<RsNxsMsg *>& messages)
 
     // store these for tick() to pick them up
     for(; vit != messages.end(); vit++)
-        mReceivedMsgs.push_back(*vit);
+    {
+    	RsNxsMsg* msg = *vit;
+
+    	NxsMsgPendingVect::iterator it =
+    			std::find(mMsgPendingValidate.begin(), mMsgPendingValidate.end(), getMsgIdPair(*msg));
+
+    	// if we have msg already just delete it
+    	if(it == mMsgPendingValidate.end())
+    		mReceivedMsgs.push_back(msg);
+    	else
+    		delete msg;
+    }
 
 }
 
@@ -1265,7 +1289,6 @@ bool RsGenExchange::processGrpMask(const RsGxsGroupId& grpId, ContentValue &grpC
     RsGxsGrpMetaData* grpMeta = NULL;
     bool ok = false;
 
-
     std::map<RsGxsGroupId, RsGxsGrpMetaData* > grpMetaMap;
     std::map<RsGxsGroupId, RsGxsGrpMetaData* >::iterator mit;
     grpMetaMap.insert(std::make_pair(grpId, (RsGxsGrpMetaData*)(NULL)));
@@ -1312,130 +1335,172 @@ bool RsGenExchange::processGrpMask(const RsGxsGroupId& grpId, ContentValue &grpC
 
 void RsGenExchange::publishMsgs()
 {
+
 	RsStackMutex stack(mGenMtx);
 
+	// stick back msgs pending signature
+	typedef std::map<uint32_t, GxsPendingSignItem<RsGxsMsgItem*, uint32_t> > PendSignMap;
+
+	PendSignMap::iterator sign_it = mMsgPendingSign.begin();
+
+	for(; sign_it != mMsgPendingSign.end(); sign_it++)
+	{
+		GxsPendingSignItem<RsGxsMsgItem*, uint32_t>& item = sign_it->second;
+		mMsgsToPublish.insert(std::make_pair(sign_it->first, item.mItem));
+	}
+
 	std::map<uint32_t, RsGxsMsgItem*>::iterator mit = mMsgsToPublish.begin();
 
-        for(; mit != mMsgsToPublish.end(); mit++)
+	for(; mit != mMsgsToPublish.end(); mit++)
 	{
-	    std::cerr << "RsGenExchange::publishMsgs() Publishing a Message";
+		std::cerr << "RsGenExchange::publishMsgs() Publishing a Message";
-	    std::cerr << std::endl;
+		std::cerr << std::endl;
 
-            RsNxsMsg* msg = new RsNxsMsg(mServType);
+		RsNxsMsg* msg = new RsNxsMsg(mServType);
-            RsGxsMsgItem* msgItem = mit->second;
+		RsGxsMsgItem* msgItem = mit->second;
+		const uint32_t& token = mit->first;
 
-            msg->grpId = msgItem->meta.mGroupId;
+		msg->grpId = msgItem->meta.mGroupId;
 
-            uint32_t size = mSerialiser->size(msgItem);
+		uint32_t size = mSerialiser->size(msgItem);
-            char* mData = new char[size];
+		char* mData = new char[size];
-            bool serialOk = false;
-            bool createOk = false;
-            serialOk = mSerialiser->serialise(msgItem, mData, &size);
 
-            if(serialOk)
+		bool serialOk = false;
-            {
-                msg->msg.setBinData(mData, size);
 
-                // now create meta
+		// for fatal sign creation
-                msg->metaData = new RsGxsMsgMetaData();
+		bool createOk = false;
-                *(msg->metaData) = msgItem->meta;
 
-                // assign time stamp
+		// if sign requests to try later
-                msg->metaData->mPublishTs = time(NULL);
+		bool tryLater = false;
 
-                // now intialise msg (sign it)
+		serialOk = mSerialiser->serialise(msgItem, mData, &size);
-                createOk = createMessage(msg);
-                RsGxsMessageId msgId;
-                RsGxsGroupId grpId = msgItem->meta.mGroupId;
 
-                bool msgDoesnExist = false, validSize = false;
+		if(serialOk)
+		{
+			msg->msg.setBinData(mData, size);
 
-                if(createOk)
+			// now create meta
-                {
+			msg->metaData = new RsGxsMsgMetaData();
+			*(msg->metaData) = msgItem->meta;
 
-                    GxsMsgReq req;
+			// assign time stamp
-                    std::vector<RsGxsMessageId> msgV;
+			msg->metaData->mPublishTs = time(NULL);
-                    msgV.push_back(msg->msgId);
+
-                    GxsMsgMetaResult res;
+			// now intialise msg (sign it)
-                    req.insert(std::make_pair(msg->grpId, msgV));
+			uint8_t createReturn = createMessage(msg);
-                    mDataStore->retrieveGxsMsgMetaData(req, res);
+
-                    msgDoesnExist = res[grpId].empty();
+			if(createReturn == CREATE_FAIL)
+			{
+				createOk = false;
+			}
+			else if(createReturn == CREATE_FAIL_TRY_LATER)
+			{
+				PendSignMap::iterator pit = mMsgPendingSign.find(token);
+				tryLater = true;
+
+				// add to queue of messages waiting for a successful
+				// sign attempt
+				if(pit == mMsgPendingSign.end())
+				{
+					GxsPendingSignItem<RsGxsMsgItem*, uint32_t> gsi(msgItem, token);
+					mMsgPendingSign.insert(std::make_pair(token, gsi));
+				}
+				else
+				{
+					// remove from attempts queue if over sign
+					// attempts limit
+					if(pit->second.mAttempts == SIGN_MAX_ATTEMPTS)
+					{
+						mMsgPendingSign.erase(token);
+						tryLater = false;
+					}
+					else
+					{
+						pit->second.mAttempts++;
+					}
+				}
+
+				createOk = false;
+			}
+			else if(createReturn == CREATE_SUCCESS)
+			{
+				createOk = true;
+
+				// erase from queue if it exists
+				mMsgPendingSign.erase(token);
+			}
+			else // unknown return, just fail
+				createOk = false;
+
+
+
+			RsGxsMessageId msgId;
+			RsGxsGroupId grpId = msgItem->meta.mGroupId;
+
+			bool validSize = false;
+
+			// check message not over single msg storage limit
+			if(createOk)
+			{
+				validSize = mDataStore->validSize(msg);
+			}
+
+			if(createOk && validSize)
+			{
+				// empty orig msg id means this is the original
+				// msg
+				if(msg->metaData->mOrigMsgId.empty())
+				{
+					msg->metaData->mOrigMsgId = msg->metaData->mMsgId;
+				}
+
+				// now serialise meta data
+				size = msg->metaData->serial_size();
+
+				char* metaDataBuff = new char[size];
+				bool s = msg->metaData->serialise(metaDataBuff, &size);
+				s &= msg->meta.setBinData(metaDataBuff, size);
+
+				msg->metaData->mMsgStatus = GXS_SERV::GXS_MSG_STATUS_UNPROCESSED | GXS_SERV::GXS_MSG_STATUS_UNREAD;
+				msgId = msg->msgId;
+				grpId = msg->grpId;
+				mDataAccess->addMsgData(msg);
+
+				delete[] metaDataBuff;
+
+				// add to published to allow acknowledgement
+				mMsgNotify.insert(std::make_pair(mit->first, std::make_pair(grpId, msgId)));
+				mDataAccess->updatePublicRequestStatus(mit->first, RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE);
+			}
+			else
+			{
+				// delete msg if create msg not ok
+				delete msg;
+
+				if(!tryLater)
+					mDataAccess->updatePublicRequestStatus(mit->first,
+							RsTokenService::GXS_REQUEST_V2_STATUS_FAILED);
 
 #ifdef GEN_EXCH_DEBUG
-                    if (!msgDoesnExist)
+				std::cerr << "RsGenExchange::publishMsgs() failed to publish msg " << std::endl;
-                    {
-                        std::cerr << "RsGenExchange::publishMsgs() msg exists already :( " << std::endl;
-                    }
 #endif
-                }
+			}
-
+		}
-                if(createOk && msgDoesnExist)
+		else
-                {
+		{
-                	validSize = mDataStore->validSize(msg);
-                }
-
-                if(createOk && msgDoesnExist && validSize)
-                {
-                    // empty orig msg id means this is the original
-                    // msg
-                    // TODO: a non empty msgid means one should at least
-                    // have the msg on disk, after which this msg is signed
-                    // based on the security settings
-                    // public grp (sign by grp public pub key, private/id: signed by
-                    // id
-                    if(msg->metaData->mOrigMsgId.empty())
-                    {
-                        msg->metaData->mOrigMsgId = msg->metaData->mMsgId;
-                    }
-
-                    // now serialise meta data
-                    size = msg->metaData->serial_size();
-                    char metaDataBuff[size];
-                    msg->metaData->serialise(metaDataBuff, &size);
-                    msg->meta.setBinData(metaDataBuff, size);
-
-                    msg->metaData->mMsgStatus = GXS_SERV::GXS_MSG_STATUS_UNPROCESSED | GXS_SERV::GXS_MSG_STATUS_UNREAD;
-                    msgId = msg->msgId;
-                    grpId = msg->grpId;
-                    mDataAccess->addMsgData(msg);
-
-
-                    // add to published to allow acknowledgement
-                    mMsgNotify.insert(std::make_pair(mit->first, std::make_pair(grpId, msgId)));
-                    mDataAccess->updatePublicRequestStatus(mit->first, RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE);
-                }
-                else
-                {
-                    // delete msg if created wasn't ok
-
-			/*******************************************************************************
-			 * NOTE: THIS LOGIC IS WRONG.
-			 * It is possible, that message creation failed because Author Keys
-			 * were not cached...  If this is the case - it should be re-tried
-			 * in a few seconds - when the cache has been loaded.
-			 ******************************************************************************/
-
-                    delete msg;
-                    mDataAccess->updatePublicRequestStatus(mit->first, RsTokenService::GXS_REQUEST_V2_STATUS_FAILED);
-
 #ifdef GEN_EXCH_DEBUG
-                    std::cerr << "RsGenExchange::publishMsgs() failed to publish msg " << std::endl;
+			std::cerr << "RsGenExchange::publishMsgs() failed to serialise msg " << std::endl;
 #endif
-                }
+		}
-            }
-            else
-            {
-#ifdef GEN_EXCH_DEBUG
-                std::cerr << "RsGenExchange::publishMsgs() failed to serialise msg " << std::endl;
-#endif
-            }
 
-            delete[] mData;
+		delete[] mData;
-            delete msgItem;
+
+		if(!tryLater)
+			delete msgItem;
 	}
 
-	// clear msg list as we're done publishing them and entries
+	// clear msg item map as we're done publishing them and all
-	// are invalid
+	// entries are invalid
 	mMsgsToPublish.clear();
 }
 
@@ -1699,10 +1764,30 @@ void RsGenExchange::processRecvdData()
 }
 
 
+
 void RsGenExchange::processRecvdMessages()
 {
     RsStackMutex stack(mGenMtx);
 
+
+    NxsMsgPendingVect::iterator pend_it = mMsgPendingValidate.begin();
+
+    for(; pend_it != mMsgPendingValidate.end();)
+    {
+    	GxsPendingSignItem<RsNxsMsg*, RsGxsGrpMsgIdPair>& gpsi = *pend_it;
+
+    	if(gpsi.mAttempts == VALIDATE_MAX_ATTEMPTS)
+    	{
+    		delete gpsi.mItem;
+    		pend_it = mMsgPendingValidate.erase(pend_it);
+    	}
+    	else
+    	{
+    		mReceivedMsgs.push_back(gpsi.mItem);
+    		pend_it++;
+    	}
+    }
+
     std::vector<RsNxsMsg*>::iterator vit = mReceivedMsgs.begin();
     GxsMsgReq msgIds;
     std::map<RsNxsMsg*, RsGxsMsgMetaData*> msgs;
@@ -1723,63 +1808,80 @@ void RsGenExchange::processRecvdMessages()
         RsNxsMsg* msg = *vit;
         RsGxsMsgMetaData* meta = new RsGxsMsgMetaData();
         bool ok = meta->deserialise(msg->meta.bin_data, &(msg->meta.bin_len));
+        msg->metaData = meta;
 
+        uint8_t validateReturn = VALIDATE_FAIL;
 
         if(ok)
         {
             std::map<RsGxsGroupId, RsGxsGrpMetaData*>::iterator mit = grpMetas.find(msg->grpId);
 
             // validate msg
-            if(mit != grpMetas.end()){
+            if(mit != grpMetas.end())
+            {
                 RsGxsGrpMetaData* grpMeta = mit->second;
-                ok = true;
+                validateReturn = validateMsg(msg, grpMeta->mGroupFlags, grpMeta->keys);
-                 msg->metaData = meta;
-                 ok &= validateMsg(msg, grpMeta->mGroupFlags, grpMeta->keys);
-               msg->metaData = NULL;
             }
-            else
-                ok = false;
 
-            if(ok)
+            if(validateReturn == VALIDATE_SUCCESS)
             {
                 meta->mMsgStatus = GXS_SERV::GXS_MSG_STATUS_UNPROCESSED | GXS_SERV::GXS_MSG_STATUS_UNREAD;
                 msgs.insert(std::make_pair(msg, meta));
                 msgIds[msg->grpId].push_back(msg->msgId);
+
+                NxsMsgPendingVect::iterator validated_entry = std::find(mMsgPendingValidate.begin(), mMsgPendingValidate.end(),
+                		getMsgIdPair(*msg));
+
+                if(validated_entry != mMsgPendingValidate.end()) mMsgPendingValidate.erase(validated_entry);
             }
         }
-
+        else
-        if(!ok)
         {
+        	validateReturn = VALIDATE_FAIL;
+        }
 
-		/*********************************************************************
+        if(validateReturn == VALIDATE_FAIL)
-		 * ONCE AGAIN - You fail to handle the case where the Key is not in the Cache.
+        {
-		 * When this happens -- and it will happen frequently!
-		 * You MUST retry.
-		 *
-		 * Suggested approach:
-		 *     1) Change validateMsg() and createMsgSignatures to return INT instead of BOOL.
-		 *     2) return -1 for fail , 0 for missing keys, 1 for success.  
-                 *     3) if missing keys, put on queue and retry in 1 or 2 seconds.
-		 *
-	  	 ************************************************************************/
-
 
 #ifdef GXS_GENX_DEBUG
             std::cerr << "failed to deserialise incoming meta, grpId: "
                     << msg->grpId << ", msgId: " << msg->msgId << std::endl;
 #endif
+
+            NxsMsgPendingVect::iterator failed_entry = std::find(mMsgPendingValidate.begin(), mMsgPendingValidate.end(),
+            		getMsgIdPair(*msg));
+
+            if(failed_entry != mMsgPendingValidate.end()) mMsgPendingValidate.erase(failed_entry);
             delete msg;
-            delete meta;
+
+
+        }
+        else if(validateReturn == VALIDATE_FAIL_TRY_LATER)
+        {
+
+        	RsGxsGrpMsgIdPair id;
+        	id.first = msg->grpId;
+        	id.second = msg->msgId;
+
+        	// first check you haven't made too many attempts
+
+        	NxsMsgPendingVect::iterator vit = std::find(
+        			mMsgPendingValidate.begin(), mMsgPendingValidate.end(), id);
+
+        	if(vit == mMsgPendingValidate.end())
+        	{
+        		GxsPendingSignItem<RsNxsMsg*, RsGxsGrpMsgIdPair> item(msg, id);
+        		mMsgPendingValidate.push_back(item);
+        	}else
+        	{
+				vit->mAttempts++;
+        	}
         }
     }
 
-    std::map<RsGxsGroupId, RsGxsGrpMetaData*>::iterator mit = grpMetas.begin();
+    // clean up resources from group meta retrieval
-
+    freeAndClearContainerResource<std::map<RsGxsGroupId, RsGxsGrpMetaData*>,
-    // clean up resources
+    	RsGxsGrpMetaData*>(grpMetas);
-    for(; mit != grpMetas.end(); mit++)
-    {
-        delete mit->second;
-    }
 
     if(!msgIds.empty())
     {

libretroshare/src/gxs/rsgenexchange.h

@@ -38,11 +38,35 @@
 #include "retroshare/rsgxsservice.h"
 #include "serialiser/rsnxsitems.h"
 
+template<class GxsItem, typename Identity = std::string>
+class GxsPendingSignItem
+{
+public:
+	GxsPendingSignItem(GxsItem item, Identity id) :
+		mItem(item), mId(id), mAttempts(0)
+	{}
+
+	GxsPendingSignItem(const GxsPendingSignItem& gpsi)
+	{
+		this->mItem = gpsi.mItem;
+		this->mId = gpsi.mId;
+		this->mAttempts = gpsi.mAttempts;
+	}
+
+	bool operator==(const Identity& id)
+	{
+		return this->mId == id;
+	}
+
+	GxsItem mItem;
+	Identity mId;
+	uint8_t mAttempts;
+};
+
 typedef std::map<RsGxsGroupId, std::vector<RsGxsMsgItem*> > GxsMsgDataMap;
 typedef std::map<RsGxsGroupId, RsGxsGrpItem*> GxsGroupDataMap;
 typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsGxsMsgItem*> > GxsMsgRelatedDataMap;
 
-
 /*!
  * This should form the parent class to \n
  * all gxs services. This provides access to service's msg/grp data \n
@@ -69,6 +93,9 @@ class RsGenExchange : public RsNxsObserver, public RsThread, public RsGxsIface
 {
 public:
 
+	/// used by class derived for RsGenExchange to indicate if service create passed or not
+	enum { SERVICE_CREATE_SUCCESS, SERVICE_CREATE_FAIL, SERVICE_FAIL_TRY_LATER } ServiceCreate_Returns;
+
     /*!
      * Constructs a RsGenExchange object, the owner ship of gds, ns, and serviceserialiser passes \n
      * onto the constructed object
@@ -130,8 +157,9 @@ public:
      * Convenience function for setting bit patterns of the individual privacy level authentication
      * policy and group options
      * @param flag the bit pattern (and policy) set for the privacy policy
-     * @param authenFlag Only the policy portion chosen will be modified with 'flag'
+     * @param authenFlag Only the policy portion chosen will be modified with 'flag',
-     * @param pos The policy portion to modify
+     * the origianl flags in the indicated bit position (pos) are over-written
+     * @param pos The policy bit portion to modify
      * @see PrivacyBitPos
      */
     static bool setAuthenPolicyFlag(const uint8_t& flag, uint32_t& authenFlag, const PrivacyBitPos& pos);
@@ -234,6 +262,7 @@ public:
 
 
     bool subscribeToGroup(uint32_t& token, const RsGxsGroupId& grpId, bool subscribe);
+
 protected:
 
     /*!
@@ -355,7 +384,7 @@ protected:
      * call is blocking retrieval from underlying db
      * @warning under normal circumstance a service should not need this
      * @param grpId the id of the group to retrieve keys for
-     * @param keys this is set to the retrieved keys
+     * @param keys set to the retrieved keys
      * @return false if group does not exist or grpId is empty
      */
     bool getGroupKeys(const RsGxsGroupId& grpId, RsTlvSecurityKeySet& keySet);
@@ -530,17 +559,21 @@ private:
      * What signatures are calculated are based on the authentication policy
      * of the service
      * @param msg the Nxs message to create
+     * CREATE_FAIL, CREATE_SUCCESS, CREATE_ID_SIGN_NOT_AVAIL
+     * @return CREATE_SUCCESS for success, CREATE_FAIL for fail,
+     * 		   CREATE_ID_SIGN_NOT_AVAIL for Id sign key not avail (but requested)
      */
-    bool createMessage(RsNxsMsg* msg);
+    int createMessage(RsNxsMsg* msg);
 
     /*!
      * convenience function to create sign
      * @param signSet signatures are stored here
      * @param msgData message data to be signed
      * @param grpMeta the meta data for group the message belongs to
-     * @return false if signature creation for any required signature fails, true otherwise
+     * @return SIGN_SUCCESS for success, SIGN_FAIL for fail,
+     * 		   ID_NOT_AVAIL for Id sign key not avail (but requested), try later
      */
-    bool createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBinaryData& msgData,
+    int createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBinaryData& msgData,
                              const RsGxsMsgMetaData& msgMeta, RsGxsGrpMetaData& grpMeta);
 
     /*!
@@ -562,9 +595,10 @@ private:
      * @param msg message to be validated
      * @param grpFlag the flag for the group the message belongs to
      * @param grpKeySet
-     * @return true if msg validates, false otherwise
+     * @return VALIDATE_SUCCESS for success, VALIDATE_FAIL for fail,
+     * 		   VALIDATE_ID_SIGN_NOT_AVAIL for Id sign key not avail (but requested)
      */
-    bool validateMsg(RsNxsMsg* msg, const uint32_t& grpFlag, RsTlvSecurityKeySet& grpKeySet);
+    int validateMsg(RsNxsMsg* msg, const uint32_t& grpFlag, RsTlvSecurityKeySet& grpKeySet);
 
     /*!
      * Checks flag against a given privacy bit block
@@ -606,12 +640,24 @@ private:
     /// authentication policy
     uint32_t mAuthenPolicy;
 
+    std::map<uint32_t, GxsPendingSignItem<RsGxsMsgItem*, uint32_t> >
+    	mMsgPendingSign;
+
+    std::vector<GxsPendingSignItem<RsNxsMsg*, RsGxsGrpMsgIdPair> > mMsgPendingValidate;
+    typedef std::vector<GxsPendingSignItem<RsNxsMsg*, RsGxsGrpMsgIdPair> > NxsMsgPendingVect;
+
+    std::map<RsGxsGroupId, GxsPendingSignItem<RsGxsGrpItem*> >
+    	mGrpPendingSign, mGrpPendingValidation;
 
 private:
 
     std::vector<RsGxsNotify*> mChanges;
     std::vector<RsGxsGroupChange*> mGroupChange;
     std::vector<RsGxsMsgChange*> mMsgChange;
+
+    const uint8_t CREATE_FAIL, CREATE_SUCCESS, CREATE_FAIL_TRY_LATER, SIGN_MAX_ATTEMPTS;
+    const uint8_t SIGN_FAIL, SIGN_SUCCESS, SIGN_FAIL_TRY_LATER;
+    const uint8_t VALIDATE_FAIL, VALIDATE_SUCCESS, VALIDATE_FAIL_TRY_LATER, VALIDATE_MAX_ATTEMPTS;
 };
 
 #endif // RSGENEXCHANGE_H

libretroshare/src/gxs/rsgxs.h

@@ -27,10 +27,6 @@
 
 #include "gxs/rsgxsdata.h"
 
-#include <inttypes.h>
-#include <string>
-#include <list>
-#include <set>
 #include <map>
 #include <vector>
 

libretroshare/src/gxs/rsgxsdataaccess.h

@@ -26,7 +26,7 @@
  *
  */
 
-#include "rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "rsgxsrequesttypes.h"
 #include "rsgds.h"
 

libretroshare/src/gxs/rsgxsnetservice.cc

@@ -1365,15 +1365,22 @@ void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item)
 
 	for(; mit != grp.end(); mit++)
 	{
-		RsNxsSyncGrpItem* gItem = new
+		RsGxsGrpMetaData* grpMeta = mit->second;
+
+		if(grpMeta->mSubscribeFlags & (GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED |
+	               GXS_SERV::GROUP_SUBSCRIBE_ADMIN) )
+		{
+			RsNxsSyncGrpItem* gItem = new
 				RsNxsSyncGrpItem(mServType);
-		gItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE;
+			gItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE;
-		gItem->grpId = mit->first;
+			gItem->grpId = mit->first;
-		gItem->publishTs = mit->second->mPublishTs;
+			gItem->publishTs = mit->second->mPublishTs;
-		gItem->PeerId(peer);
+			gItem->PeerId(peer);
-		gItem->transactionNumber = transN;
+			gItem->transactionNumber = transN;
-		itemL.push_back(gItem);
+			itemL.push_back(gItem);
-		delete mit->second; // release resource
+		}
+
+		delete grpMeta; // release resource
 	}
 
 	tr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM;

libretroshare/src/gxs/rsgxsnetservice.h

@@ -119,13 +119,13 @@ typedef std::map<std::string, TransactionIdMap > TransactionsPeerMap;
  * This class implements the RsNetWorkExchangeService
  * using transactions to handle synchrnisation of Nxs items between
  * peers in a network
- * Transactions requires the maintenance of several states between peers, and whether
+ * Transactions requires the maintenance of several states between peers
  *
  * Thus a data structure maintains: peers, and their active transactions
  * Then for each transaction it needs to be noted if this is an outgoing or incoming transaction
  * Outgoing transaction are in 3 different states:
  *   1. START 2. INITIATED 3. SENDING 4. END
- * Incoming transaction are also in 3 different states
+ * Incoming transaction are in 3 different states
  *   1. START 2. RECEIVING 3. END
  */
 class RsGxsNetService : public RsNetworkExchangeService, public p3ThreadedService,

libretroshare/src/gxs/rsgxsrequesttypes.h

@@ -26,7 +26,7 @@
  *
  */
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "gxs/rsgds.h"
 
 class GxsRequest

libretroshare/src/gxs/rsgxsutil.h

@@ -0,0 +1,59 @@
+/*
+ * libretroshare/src/gxs: rsgxsutil.h
+ *
+ * RetroShare C++ Interface. Generic routines that are useful in GXS
+ *
+ * Copyright 2013-2013 by Christopher Evi-Parker
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef GXSUTIL_H_
+#define GXSUTIL_H_
+
+#include <vector>
+
+/*!
+ * Handy function for cleaning out meta result containers
+ * @param container
+ */
+template <class Container, class Item>
+void freeAndClearContainerResource(Container container)
+{
+	typename Container::iterator meta_it = container.begin();
+
+	for(; meta_it != container.end(); meta_it++)
+	{
+		delete meta_it->second;
+
+	}
+	container.clear();
+}
+
+inline RsGxsGrpMsgIdPair getMsgIdPair(RsNxsMsg& msg)
+{
+	return RsGxsGrpMsgIdPair(std::make_pair(msg.grpId, msg.msgId));
+}
+
+inline RsGxsGrpMsgIdPair getMsgIdPair(RsGxsMsgItem& msg)
+{
+	return RsGxsGrpMsgIdPair(std::make_pair(msg.meta.mGroupId, msg.meta.mMsgId));
+}
+
+
+#endif /* GXSUTIL_H_ */

libretroshare/src/libretroshare.pro

@@ -5,7 +5,7 @@ TARGET = retroshare
 
 CONFIG += test_voip 
 
-# GXS Stuff.
+ #GXS Stuff.
 # This should be disabled for releases until further notice.
 #CONFIG += gxs debug
 
@@ -22,7 +22,7 @@ profiling {
 #QMAKE_CFLAGS += -Werror
 #QMAKE_CXXFLAGS += -Werror
 
-#CONFIG += debug
+CONFIG += debug
 debug {
 #	DEFINES *= DEBUG
 #	DEFINES *= OPENDHT_DEBUG DHT_DEBUG CONN_DEBUG DEBUG_UDP_SORTER P3DISC_DEBUG DEBUG_UDP_LAYER FT_DEBUG EXTADDRSEARCH_DEBUG
@@ -627,11 +627,12 @@ gxs {
 		gxs/rsgenexchange.h \
 		gxs/rsnxsobserver.h \
 		gxs/rsgxsdata.h \
-		gxs/rstokenservice.h \
+		retroshare/rstokenservice.h \
 		gxs/rsgxsdataaccess.h \
 		retroshare/rsgxsservice.h \
 		serialiser/rsgxsitems.h \
 		util/retrodb.h \
+		gxs/rsgxsutil.h \
 		util/contentvalue.h \
 		gxs/gxssecurity.h \
 		gxs/rsgxsifacehelper.h \

libretroshare/src/retroshare/rsgxschannels.h

@@ -30,7 +30,7 @@
 #include <string>
 #include <list>
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 #include "retroshare/rsgxscommon.h"
 

libretroshare/src/retroshare/rsgxscircles.h

@@ -31,7 +31,7 @@
 #include <list>
 #include <set>
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 
 #include "retroshare/rsidentity.h"

libretroshare/src/retroshare/rsgxsforums.h

@@ -30,7 +30,7 @@
 #include <string>
 #include <list>
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 
 

libretroshare/src/retroshare/rsgxsservice.h

@@ -2,7 +2,7 @@
 #define RSGXSSERVICE_H
 
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 
 typedef std::map<RsGxsGroupId, std::vector<RsMsgMetaData> > GxsMsgMetaMap;
 typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsMsgMetaData> > GxsMsgRelatedMetaMap;

libretroshare/src/retroshare/rsidentity.h

@@ -30,7 +30,7 @@
 #include <string>
 #include <list>
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 
 /* The Main Interface Class - for information about your Peers */

libretroshare/src/retroshare/rsposted.h

@@ -30,7 +30,7 @@
 #include <inttypes.h>
 #include <string>
 #include <list>
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 
 class RsPosted;

libretroshare/src/retroshare/rswiki.h

@@ -30,7 +30,7 @@
 #include <string>
 #include <list>
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 
 /* The Main Interface Class - for information about your Peers */

libretroshare/src/retroshare/rswire.h

@@ -30,7 +30,7 @@
 #include <string>
 #include <list>
 
-#include "gxs/rstokenservice.h"
+#include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
 
 

retroshare-gui/src/util/TokenQueue.h

@@ -30,7 +30,7 @@
 #include <string>
 #include <sys/time.h>
 
-#include <gxs/rstokenservice.h>
+#include <retroshare/rstokenservice.h>
 
 
 #define COMPLETED_REQUEST		4