implemented most of rsdataservice,

updated rsnxsitems with discrete msgs and grp item rather than collection of msgs/grps
added flags header file 
added first declaration of implemented genexchange service (client API interface)

git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-new_cache_system@5180 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/gxs/rsdataservice.cc

@@ -0,0 +1,509 @@
+#include <fstream>
+
+#include "rsdataservice.h"
+
+#define MSG_TABLE_NAME std::string("MESSAGES")
+#define GRP_TABLE_NAME std::string("GROUPS")
+
+
+// generic
+#define KEY_NXS_FILE std::string("msgFile")
+#define KEY_NXS_FILE_OFFSET std::string("fileOffset")
+#define KEY_NXS_LEN std::string("msgLen")
+#define KEY_NXS_IDENTITY std::string("identity")
+#define KEY_GRP_ID std::string("grpId")
+#define KEY_IDENTITY_SIGN std::string("idSign")
+#define KEY_TIME_STAMP std::string("timeStamp")
+#define KEY_NXS_FLAGS std::string("flags")
+
+// grp table columns
+#define KEY_ADMIN_SIGN std::string("adminSign")
+#define KEY_PUB_PUBLISH_KEY std::string("pubPublishhKey")
+#define KEY_PUB_ADMIN_KEY std::string("pubAdminKey")
+#define KEY_PRIV_ADMIN_KEY std::string("privAdminKey")
+#define KEY_PRIV_PUBLISH_KEY std::string("privPublishKey")
+#define KEY_GRP_FILE std::string("grpFile")
+
+
+// msg table columns
+#define KEY_PUBLISH_SIGN std::string("publishSign")
+#define KEY_MSG_ID std::string("msgId")
+
+
+// grp col numbers
+#define COL_GRP_ID 1
+#define COL_ADMIN_SIGN 2
+#define COL_PUB_PUBLISH_KEY 10
+#define COL_PUB_ADMIN_KEY 11
+#define COL_PRIV_ADMIN_KEY 12
+#define COL_PRIV_PUBLISH_KEY 13
+
+
+// msg col numbers
+#define COL_MSG_ID 1
+#define COL_PUBLISH_SIGN 2
+
+// generic col numbers
+#define COL_NXS_FILE 3
+#define COL_NXS_FILE_OFFSET 4
+#define COL_NXS_LEN 5
+#define COL_TIME_STAMP 6
+#define COL_NXS_FLAGS 7
+#define COL_IDENTITY_SIGN 8
+#define COL_IDENTITY 9
+
+
+RsDataService::RsDataService(const std::string &serviceDir, const std::string &dbName, uint16_t serviceType,
+                             RsGxsSearchModule *mod)
+    : mServiceDir(serviceDir), mDbName(dbName), mServType(serviceType){
+
+
+    // initialise database
+    remove("RsFileGdp_DataBase");
+    mDb = new RetroDb(dbName, RetroDb::OPEN_READWRITE_CREATE);
+
+    // create table for msgs
+    mDb->execSQL("CREATE TABLE " + MSG_TABLE_NAME + "(" + KEY_MSG_ID
+                 + " TEXT PRIMARY KEY ASC," + KEY_GRP_ID +  " TEXT," + KEY_NXS_FLAGS + " INT,"
+                  + KEY_TIME_STAMP + " INT," + KEY_PUBLISH_SIGN + " BLOB," + KEY_NXS_IDENTITY + " TEXT,"
+                 + KEY_IDENTITY_SIGN + " BLOB," + KEY_NXS_FILE + " TEXT,"+ KEY_NXS_FILE_OFFSET + " INT,"
+                 + KEY_NXS_LEN+ " INT);");
+
+    // create table for grps
+    mDb->execSQL("CREATE TABLE " + GRP_TABLE_NAME + "(" + KEY_GRP_ID +
+                 " TEXT PRIMARY KEY ASC," + KEY_TIME_STAMP + " INT," +
+                 KEY_ADMIN_SIGN + " BLOB," + KEY_PUB_ADMIN_KEY + " BLOB,"
+                 + KEY_PUB_PUBLISH_KEY + " BLOB," + KEY_PRIV_ADMIN_KEY +
+                 " BLOB," + KEY_PRIV_PUBLISH_KEY + " BLOB," + KEY_NXS_FILE +
+                  " TEXT," + KEY_NXS_FILE_OFFSET + " INT," + KEY_NXS_LEN + " INT,"
+                  + KEY_NXS_IDENTITY + " TEXT," + KEY_NXS_FLAGS + " INT," + KEY_IDENTITY_SIGN + " BLOB);");
+
+
+    msgColumns.push_back(KEY_MSG_ID); msgColumns.push_back(KEY_PUBLISH_SIGN);  msgColumns.push_back(KEY_NXS_FILE);
+    msgColumns.push_back(KEY_NXS_FILE_OFFSET); msgColumns.push_back(KEY_NXS_LEN); msgColumns.push_back(KEY_TIME_STAMP);
+    msgColumns.push_back(KEY_NXS_FLAGS); msgColumns.push_back(KEY_NXS_IDENTITY); msgColumns.push_back(KEY_IDENTITY_SIGN);
+
+    grpColumns.push_back(KEY_GRP_ID); grpColumns.push_back(KEY_ADMIN_SIGN); grpColumns.push_back(KEY_NXS_FILE);
+    grpColumns.push_back(KEY_NXS_FILE_OFFSET); grpColumns.push_back(KEY_NXS_LEN); grpColumns.push_back(KEY_TIME_STAMP);
+    grpColumns.push_back(KEY_NXS_FLAGS); grpColumns.push_back(KEY_NXS_IDENTITY); grpColumns.push_back(KEY_IDENTITY_SIGN);
+    grpColumns.push_back(KEY_PUB_PUBLISH_KEY); grpColumns.push_back(KEY_PUB_ADMIN_KEY); grpColumns.push_back(KEY_PRIV_ADMIN_KEY);
+    grpColumns.push_back(KEY_PRIV_PUBLISH_KEY); grpColumns.push_back(KEY_GRP_FILE);
+}
+
+
+RsNxsGrp* RsDataService::getGroup(RetroCursor &c){
+
+    /*!
+     * grpId, pub admin and pub publish key
+     * necessary for successful group
+     */
+
+    RsNxsGrp* grp = new RsNxsGrp(mServType);
+    bool ok = true;
+
+    uint32_t offset = 0;
+    // declare members of nxsgrp
+    char* data = NULL;
+    uint32_t data_len = 0;
+
+    // grpId
+    c.getString(COL_GRP_ID, grp->grpId);
+    ok &= !grp->grpId.empty();
+
+    // identity if any
+    c.getString(COL_IDENTITY, grp->identity);
+    grp->timeStamp = c.getInt64(COL_TIME_STAMP);
+
+    if(!grp->identity.empty() && ok){
+        offset = 0;
+        data = (char*)c.getData(COL_IDENTITY_SIGN, data_len);
+        if(data){
+            grp->adminSign.GetTlv(data, data_len, &offset);
+        }
+    }
+
+    grp->grpFlag = c.getInt32(COL_NXS_FLAGS);
+
+
+    offset = 0; data = NULL; data_len = 0;
+
+    data = c.getData(COL_PRIV_ADMIN_KEY, data_len);
+    if(data){
+        ok &= grp->keys.SetTlv(data, data_len, &offset);
+    }
+
+    std::string grpFile;
+    c.getString(COL_NXS_FILE, grpFile);
+    data_len = c.getInt32(COL_NXS_LEN);
+    ok &= !grpFile.empty();
+    /* now retrieve grp data from file */
+
+    if(ok){
+        offset = c.getInt32(COL_NXS_FILE_OFFSET);
+        std::ifstream istrm(grpFile.c_str());
+        istrm.open(grpFile.c_str(), std::ios::binary);
+        istrm.seekg(offset, std::ios::beg);
+        char grp_data[data_len];
+        istrm.read(grp_data, data_len);
+        istrm.close();
+        offset = 0;
+        ok &= grp->grp.SetTlv(grp_data, data_len, &offset);
+    }
+
+    if(ok)
+        return grp;
+    else
+        delete grp;
+
+    return NULL;
+}
+
+
+RsNxsMsg* RsDataService::getMessage(RetroCursor &c){
+
+    RsNxsMsg* msg = new RsNxsMsg(mServType);
+
+    bool ok = true;
+    uint32_t data_len = 0,
+    offset = 0;
+    char* data = NULL;
+    c.getString(COL_GRP_ID, msg->grpId);
+    c.getString(COL_MSG_ID, msg->msgId);
+
+    ok &= (!msg->grpId.empty()) && (!msg->msgId.empty());
+
+    c.getString(COL_IDENTITY, msg->identity);
+
+    if(!msg->identity.empty()){
+        data = (char*)c.getData(COL_IDENTITY_SIGN, data_len);
+        msg->idSign.SetTlv(data, data_len, &offset);
+    }
+
+    msg->msgFlag = c.getInt32(COL_NXS_FLAGS);
+    msg->timeStamp = c.getInt32(COL_TIME_STAMP);
+
+    offset = 0; data_len = 0;
+    if(ok){
+
+        data = c.getData(COL_PUBLISH_SIGN, data_len);
+        if(data)
+            msg->publishSign.SetTlv(data, data_len, &offset);
+
+    }
+
+    std::string grpFile;
+    c.getString(COL_NXS_FILE, grpFile);
+    data_len = c.getInt32(COL_NXS_LEN);
+    ok &= !msgFile.empty();
+    /* now retrieve grp data from file */
+
+    if(ok){
+        offset = c.getInt32(COL_NXS_FILE_OFFSET);
+        std::ifstream istrm(grpFile.c_str());
+        istrm.open(grpFile.c_str(), std::ios::binary);
+        istrm.seekg(offset, std::ios::beg);
+        char grp_data[data_len];
+        istrm.read(grp_data, data_len);
+        istrm.close();
+        offset = 0;
+        ok &= grp->grp.SetTlv(grp_data, data_len, &offset);
+    }
+
+    if(ok)
+        return msg;
+    else
+        delete msg;
+
+    return NULL;
+}
+
+int RsDataService::storeMessage(std::set<RsNxsMsg *> &msg){
+
+
+    std::set<RsNxsMsg*>::iterator sit = msg.begin();
+
+    mDb->execSQL("BEGIN;");
+
+
+    for(; sit != msg.end(); sit++){
+
+        RsNxsMsg* msgPtr = *sit;
+        std::string msgFile = mServiceDir + "/" + msgPtr->grpId + "-msgs";
+        std::fstream ostrm(msgFile.c_str(), std::ios::binary | std::ios::app | std::ios::out);
+        ostrm.seekg(0, std::ios::end); // go to end to append
+        int32_t offset = ostrm.tellg(); // get fill offset
+
+        ContentValue cv;
+        cv.put(KEY_MSG_ID, msgPtr->msgId);
+        cv.put(KEY_GRP_ID, msgPtr->grpId);
+        char pubSignData[msgPtr->publishSign.TlvSize()];
+        cv.put(KEY_PUBLISH_SIGN, msgPtr->publishSign.TlvSize(), pubSignData);
+
+        if(! (msgPtr->identity.empty()) ){
+            char idSignData[msgPtr->idSign.TlvSize()];
+            cv.put(KEY_IDENTITY_SIGN, msgPtr->idSign.TlvSize(), idSignData);
+            cv.put(KEY_NXS_IDENTITY, msgPtr->identity);
+        }
+
+        cv.put(KEY_NXS_FLAGS, (int32_t) msgPtr->msgFlag);
+        cv.put(KEY_TIME_STAMP, (int32_t) msgPtr->timeStamp);
+
+        char msgData[msgPtr->msg.TlvSize()];
+        msgPtr->msg.SetTlv(msgData, msgPtr->msg.TlvSize(), offset);
+        ostrm.write(msgData, msgPtr->msg.TlvSize());
+        ostrm.close();
+
+        cv.put(KEY_NXS_FILE_OFFSET, offset);
+
+        mDb->sqlInsert(MSG_TABLE_NAME, "", cv);
+    }
+
+    return mDb->execSQL("COMMIT;");;
+}
+
+
+int RsDataService::storeGroup(std::set<RsNxsGrp *> &grp){
+
+
+    std::set<RsNxsGrp*>::iterator sit = grp.begin();
+
+    mDb->execSQL("BEGIN;");
+
+    for(; sit != grp.end(); sit++){
+
+        RsNxsGrp* grpPtr = *sit;
+
+        std::string grpFile = mServiceDir + "/" + msgPtr->grpId;
+        std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::app | std::ios::out);
+        ostrm.seekg(0, std::ios::end); // go to end to append
+        int32_t offset = ostrm.tellg(); // get fill offset
+
+        ContentValue cv;
+        cv.put(KEY_GRP_ID, grpPtr->grpId);
+        cv.put(KEY_NXS_FLAGS, (int32_t)grpPtr->grpFlag);
+        cv.put(KEY_TIME_STAMP, (int32_t)grpPtr->timeStamp);
+
+        if(! (grpPtr->identity.empty()) ){
+            cv.put(KEY_NXS_IDENTITY, grpPtr->identity);
+
+            char idSignData[msgPtr->idSign.TlvSize()];
+            cv.put(KEY_IDENTITY_SIGN, grpPtr->idSign.TlvSize(), idSignData);
+        }
+
+        char adminSignData[grpPtr->adminSign.TlvSize()];
+        grpPtr->adminSign.SetTlv(adminSignData, grpPtr->adminSign.TlvSize(), offset);
+        cv.put(KEY_ADMIN_SIGN, grpPtr->adminSign.TlvSize());
+
+        char keySetData[grpPtr->keys.TlvSize()];
+        grpPtr->keys.SetTlv(data, grpPtr->keys.TlvSize(), offset);
+        cv.put(KEY_KEY_SET, grpPtr->keys.TlvSize(), keySetData);
+
+        char grpData[grpPtr->grp.TlvSize()];
+        msgPtr->msg.SetTlv(grpData, grpPtr->grp.TlvSize(), offset);
+        ostrm.write(grpData, grpPtr->grp.TlvSize());
+        ostrm.close();
+    }
+
+    return mDb->execSQL("COMMIT;");
+}
+
+int RsDataService::retrieveGrps(std::map<std::string, RsNxsGrp*> &grp, bool cache){
+
+    RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpColumns, "", "");
+
+    if(c){
+
+        bool valid = c->moveToFirst();
+
+        while(valid){
+            RsNxsGrp* g = getGroup(*c);
+
+
+            // only add the latest grp info
+            bool exists = grp.find(g->grpId) != grp.end();
+            if(exists){
+
+                if(grp[g->grpId]->timeStamp < g->timeStamp){
+                    delete grp[g->grpId];
+                    grp.insert(g);
+                }else{
+                    delete g;
+                }
+            }else{
+                grp.insert(g);
+            }
+
+            valid = c->moveToNext();
+        }
+
+        delete c;
+        return 1;
+    }else{
+        return 0;
+    }
+}
+
+int RsDataService::retrieveMsgs(const std::string &grpId, std::map<std::string, RsGxsMsg *> msg, bool cache){
+
+    RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, msgColumns, KEY_GRP_ID+ "=" + grpId, "");
+
+    if(c){
+
+        bool valid = c->moveToFirst();
+        while(valid){
+            RsNxsMsg* m = getMessage(*c);
+
+            // only add the latest grp info
+            bool exists = grp.find(g->grpId) != grp.end();
+            if(exists){
+
+                if(grp[m->msgId]->timeStamp < m->timeStamp){
+                    delete msg[m->msgId];
+                    msg[m->msgId] = m;
+                }else{
+                    delete m;
+                }
+            }else{
+                msg[m->msgId] = m;
+            }
+
+            valid = c->moveToNext();
+        }
+
+        delete c;
+        return 1;
+    }else{
+        return 0;
+    }
+}
+
+int RsDataService::retrieveMsgVersions(const std::string &grpId, const std::string msgId,
+                                       std::set<RsNxsMsg *> msg, bool cache){
+
+
+    std::string selection = KEY_GRP_ID + "=" + grpId + "," + KEY_MSG_ID + "=" + msgId;
+    RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, msgColumns, selection, "");
+
+
+    if(c){
+
+        bool valid = c->moveToFirst();
+        while(valid){
+            RsNxsMsg* m = getMessage(*c);
+
+            if(m)
+                msg.insert(m);
+
+            valid = c->moveToNext();
+        }
+
+        delete c;
+        return 1;
+    }else{
+        return 0;
+    }
+
+}
+
+int RsDataService::retrieveGrpVersions(const std::string &grpId, std::set<RsGxsGroup *> &grp){
+
+    std::string selection = KEY_GRP_ID + "=" + grpId;
+    RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, msgColumns, selection, "");
+
+    if(c){
+
+        bool valid = c->moveToFirst();
+        while(valid){
+            RsNxsGrp* g = getGroup(*c);
+
+            if(g)
+                grp.insert(g);
+
+            valid = c->moveToNext();
+        }
+
+        delete c;
+        return 1;
+    }else{
+        return 0;
+    }
+}
+
+RsNxsGrp* RsDataService::retrieveGrpVersion(const RsGxsGrpId &grpId){
+
+    std::set<RsNxsGrp*> grps;
+    retrieveGrpVersions(grpId.grpId, grps, false);
+    RsNxsGrp* grp = NULL;
+
+    if(!grps.empty()){
+
+        // find grp with comparable sign
+        std::set<RsNxsGrp*>::iterator sit = grps.begin();
+
+        for(; sit != grps.end(); sit++){
+            grp = *sit;
+            if(grp->adminSign == grpId.adminSign){
+                break;
+            }
+            grp = NULL;
+        }
+
+        if(grp){
+            grps.erase(grp);
+            // release memory for non matching grps
+            for(sit = grps.begin(); sit != grps.end(); sit++)
+                delete *sit;
+        }
+
+    }
+
+    return grp;
+}
+
+RsNxsMsg* RsDataService::retrieveMsgVersion(const RsGxsMsgId &msgId){
+
+    std::set<RsNxsMsg*> msgs;
+    retrieveMsgVersions(msgId.grpId, msgId.msgId, msgs);
+    RsNxsMsg* msg = NULL;
+
+    if(!msgs.empty()){
+
+        std::set<RsNxsMsg*>::iterator sit = msgs.begin();
+
+        for(; sit != msgs.end(); sit++){
+
+            msg = *sit;
+            if(msg->idSign == msgId.idSign)
+                break;
+
+            msg = NULL;
+        }
+
+        if(msg){
+            msgs.erase(msg);
+
+            for(sit = msgs.begin(); sit != msgs.end(); sit++)
+                delete *sit;
+        }
+    }
+
+
+    return msg;
+}
+
+uint32_t RsDataService::cacheSize() const {
+    return 0;
+}
+
+int RsDataService::setCacheSize(uint32_t size) {
+    return 0;
+}
+
+int RsDataService::searchGrps(RsGxsSearch *search, std::list<RsGxsSrchResGrpCtx *> &result) {
+
+    return 0;
+}
+
+int RsDataService::searchMsgs(RsGxsSearch *search, std::list<RsGxsSrchResMsgCtx *> &result) {
+    return 0;
+}

libretroshare/src/gxs/rsdataservice.cpp

@@ -1,154 +0,0 @@
-#include "rsdataservice.h"
-
-#define MSG_TABLE_NAME std::string("MESSAGES")
-#define GRP_TABLE_NAME std::string("GROUPS")
-
-
-// generic
-#define NXS_FILE std::string("msgFile")
-#define NXS_FILE_OFFSET std::string("fileOffset")
-#define NXS_LEN std::string("msgLen")
-#define NXS_IDENTITY std::string("identity")
-#define GRP_ID std::string("grpId")
-#define IDENTITY_SIGN std::string("idSign")
-#define TIME_STAMP std::string("timeStamp")
-#define NXS_FLAGS std::string("flags")
-
-// grp table columns
-#define ADMIN_SIGN std::string("adminSign")
-#define PUB_PUBLISH_KEY std::string("pubPublishhKey")
-#define PUB_ADMIN_KEY std::string("pubAdminKey")
-#define PRIV_ADMIN_KEY std::string("privAdminKey")
-#define PRIV_PUBLISH_KEY std::string("privPublishKey")
-#define GRP_FILE std::string("grpFile")
-
-
-// msg table columns
-#define PUBLISH_SIGN std::string("publishSign")
-#define MSG_ID std::string("msgId")
-
-
-// grp col numbers
-#define COL_GRP_ID 1
-#define COL_ADMIN_SIGN 2
-#define COL_PUB_PUBLISH_KEY 10
-#define COL_PUB_ADMIN_KEY 11
-#define COL_PRIV_ADMIN_KEY 12
-#define COL_PRIV_PUBLISH_KEY 13
-
-
-// msg col numbers
-#define COL_MSG_ID 1
-#define COL_PUBLISH_SIGN 2
-
-// generic col numbers
-#define COL_NXS_FILE 3
-#define COL_NXS_FILE_OFFSET 4
-#define COL_NXS_LEN 5
-#define COL_TIME_STAMP 6
-#define COL_FLAGS 7
-#define COL_IDENTITY_SIGN 8
-#define COL_IDENTITY 9
-
-
-RsDataService::RsDataService(const std::string &workingDir, const std::string &dbNamem, uint16_t serviceType)
-{
-
-
-    // initialise database
-    remove("RsFileGdp_DataBase");
-    mDb = new RetroDb(dbName, RetroDb::OPEN_READWRITE_CREATE);
-
-    // create table for msgs
-    mDb->execSQL("CREATE TABLE " + MSG_TABLE_NAME + "(" + MSG_ID
-                 + " TEXT PRIMARY KEY ASC," + GRP_ID +  " TEXT," + FLAGS + " INT,"
-                  + TIME_STAMP + " INT," + PUBLISH_SIGN + " BLOB," + NXS_IDENTITY + " TEXT,"
-                 + IDENTITY_SIGN + " BLOB," + NXS_FILE + " TEXT,"+ NXS_FILE_OFFSET + " INT,"
-                 + NXS_LEN+ " INT);");
-
-    // create table for grps
-    mDb->execSQL("CREATE TABLE " + GRP_TABLE_NAME + "(" + GRP_ID +
-                 " TEXT PRIMARY KEY ASC," + TIME_STAMP + " INT," +
-                 ADMIN_SIGN + " BLOB," + PUB_ADMIN_KEY + " BLOB,"
-                 + PUB_PUBLISH_KEY + " BLOB," + PRIV_ADMIN_KEY +
-                 " BLOB," + PRIV_PUBLISH_KEY + " BLOB," + NXS_FILE +
-                  " TEXT," + NXS_FILE_OFFSET + " INT," + NXS_LEN + " INT,"
-                  + NXS_IDENTITY + " TEXT," + IDENTITY_SIGN + " BLOB);");
-
-
-    msgColumns.push_back(MSG_ID); msgColumns.push_back(PUBLISH_SIGN);  msgColumns.push_back(NXS_FILE);
-    msgColumns.push_back(NXS_FILE_OFFSET); msgColumns.push_back(NXS_LEN); msgColumns.push_back(TIME_STAMP);
-    msgColumns.push_back(NXS_FLAGS); msgColumns.push_back(IDENTITY); msgColumns.push_back(IDENTITY_SIGN);
-
-    grpColumns.push_back(GRP_ID); grpColumns.push_back(ADMIN_SIGN); grpColumns.push_back(NXS_FILE);
-    grpColumns.push_back(NXS_FILE_OFFSET); grpColumns.push_back(NXS_LEN); grpColumns.push_back(TIME_STAMP);
-    grpColumns.push_back(NXS_FLAGS); grpColumns.push_back(IDENTITY); grpColumns.push_back(IDENTITY_SIGN);
-    grpColumns.push_back(PUB_PUBLISH_KEY); grpColumns.push_back(PUB_ADMIN_KEY); grpColumns.push_back(PRIV_ADMIN_KEY);
-    grpColumns.push_back(PRIV_PUBLISH_KEY); grpColumns.push_back(GRP_FILE);
-}
-
-
-
-
-RsNxsGrp* RsDataService::getGroup(RetroCursor &c){
-
-    RsNxsGrp* grp = new RsNxsGrp(mServType);
-    bool ok = true;
-
-    std::ifstream istrm;
-    uint32_t len, offset;
-    char* data = NULL;
-    uint32_t data_len = 0;
-    uint32_t offset = 0;
-    std::string grpId;
-    uint32_t timeStamp;
-    RsTlvBinaryData grpData;
-    RsTlvSecurityKey adminKey, pubKey, privAdminKey, privPubKey;
-    std::string identity;
-
-
-    c.getString(COL_GRP_ID, grpId);
-    ok &= !grpId.empty();
-    c.getString(COL_IDENTITY, identity);
-
-    timeStamp = c.getInt64(COL_TIME_STAMP);
-
-    /* get keys */
-    grp->keys.groupId = grpId;
-
-    data = c.getData(COL_PUB_ADMIN_KEY, data_len);
-    ok &= data;
-    if(data){
-        adminKey.GetTlv(data, data_len, &offset);
-        grp->keys.keys.insert(adminKey);
-    }
-    offset = 0;
-
-    data = c.getData(COL_PUB_PUBLISH_KEY, data_len);
-    ok &= data;
-    if(data){
-        pubKey.GetTlv(data, data_len, &offset);
-        grp->keys.keys.insert(pubKey);
-    }
-    offset = 0;
-
-    data = c.getData(COL_PRIV_ADMIN_KEY, data_len);
-    if(data){
-        privAdminKey.GetTlv(COL_PRIV_ADMIN_KEY, data_len, &offset);
-        grp->keys.keys.insert(privAdminKey);
-    }
-    offset = 0;
-
-    data = c.getData(COL_PRIV_PUBLISH_KEY);
-    if(data){
-        privPubKey.GetTlv(data, data_len, &offset);
-        grp->keys.keys.insert(privPubKey);
-    }
-
-    if(ok)
-        return grp;
-    else
-        delete grp;
-
-    return NULL;
-}

libretroshare/src/gxs/rsdataservice.h

@@ -9,41 +9,54 @@ class RsDataService : public RsGeneralDataService
 {
 public:
 
-    RsDataService(const std::string& workingDir, const std::string& dbName, uint16_t serviceType);
+    RsDataService(const std::string& serviceDir, const std::string& dbName, uint16_t serviceType, RsGxsSearchModule* mod = NULL);
     virtual ~RsDataService();
 
     /*!
-     * Retrieves signed message
+     * Retrieves latest version of msgs for a service
      * @param msgIds ids of messagesto retrieve
      * @param msg result of msg retrieval
      * @param cache whether to store retrieval in memory for faster later retrieval
      * @return error code
      */
-    int retrieveMsgs(const std::list<std::string>& msgIds, std::set<RsGxsMsg*> msg, bool cache);
+    int retrieveMsgs(const std::string& grpId, std::map<std::string, RsGxsMsg*> msg, bool cache);
 
     /*!
-     * Retrieves a group item by grpId
+     * Retrieves latest version of groups for a service
      * @param grpId the Id of the groups to retrieve
      * @param grp results of retrieval
      * @param cache whether to store retrieval in mem for faster later retrieval
      * @return error code
      */
-    int retrieveGrps(const std::list<std::string>& grpId, std::set<RsGxsGroup*>& grp, bool cache);
+    int retrieveGrps(std::set<std::string, RsGxsGroup*>& grp, bool cache);
 
     /*!
+     * Retrieves all the versions of a group
      * @param grpId the id of the group to get versions for
      * @param cache whether to store the result in memory
-     * @param errCode
+     * @return errCode
      */
-    int retrieveGrpVersions(const std::string& grpId, std::set<RsGxsGroup*>& grp, bool cache);
+    int retrieveGrpVersions(const std::string &grpId, std::set<RsNxsGrp *> &grp, bool cache);
 
     /*!
+     * retrieves all the versions of a message for a group
+     * @param grpId the id of the group message belongs to
      * @param msgId the id of the message to get versions for
-     * @param cache whether to store the result in memory
-     * @param errCode
+     * @return errCode
      */
-    int retrieveMsgVersions(const std::string& grpId, std::set<RsGxsGroup*> grp, bool cache);
+    int retrieveMsgVersions(const std::string &grpId, const std::string msgId, std::set<RsNxsMsg *> grp, bool cache);
 
+    /*!
+     * @param msgId the id of the message to retrieve
+     * @return NULL if message does not exist, or pointer to grp if found
+     */
+    RsNxsMsg* retrieveMsgVersion(const RsGxsMsgId& msgId);
+
+    /*!
+     * @param grpId the id of the group to retrieve
+     * @return NULL if group does not exist or pointer to grp if found
+     */
+    RsNxsGrp* retrieveGrpVersion(const RsGxsGrpId& grpId);
 
     /*!
      * allows for more complex queries specific to the service
@@ -62,10 +75,14 @@ public:
     int searchGrps(RsGxsSearch* search, std::list<RsGxsSrchResGrpCtx*>& result);
 
     /*!
-     * @return the cache size set for this RsGeneralDataService
+     * @return the cache size set for this RsGeneralDataService in bytes
      */
     uint32_t cacheSize() const;
 
+    /*!
+     * @param size size of cache to set in bytes
+     */
+    virtual int setCacheSize(uint32_t size);
 
     /*!
      * Stores a list signed messages into data store
@@ -87,13 +104,15 @@ private:
     RsNxsMsg* getMessage(RetroCursor& c);
     RsNxsGrp* getGroup(RetroCursor& c);
 
-
 private:
 
 
     RetroDb* mDb;
     std::list<std::string> msgColumns;
     std::list<std::string> grpColumns;
+
+    std::string mServiceDir, mDbName;
+    uint16_t mServType;
 };
 
 #endif // RSDATASERVICE_H

libretroshare/src/gxs/rsgds.h

@@ -69,12 +69,12 @@ public:
 
     /*!
      * Retrieves signed message
-     * @param msgIds ids of messagesto retrieve
+     * @param grpId
      * @param msg result of msg retrieval
      * @param cache whether to store retrieval in memory for faster later retrieval
      * @return error code
      */
-    virtual int retrieveMsgs(const std::list<std::string>& msgIds, std::set<RsGxsMsg*> msg, bool cache) = 0;
+    virtual int retrieveMsgs(const std::string& grpId, std::map<std::string, RsNxsMsg*> msg, bool cache) = 0;
 
     /*!
      * Retrieves a group item by grpId
@@ -83,22 +83,33 @@ public:
      * @param cache whether to store retrieval in mem for faster later retrieval
      * @return error code
      */
-    virtual int retrieveGrps(const std::list<std::string>& grpId, std::set<RsGxsGroup*>& grp, bool cache) = 0;
+    virtual int retrieveGrps(std::set<std::string, RsNxsGrp*>& grp, bool cache) = 0;
 
     /*!
      * @param grpId the id of the group to get versions for
      * @param cache whether to store the result in memory
      * @param errCode
      */
-    virtual int retrieveGrpVersions(const std::string& grpId, std::set<RsGxsGroup*>& grp, bool cache);
+    virtual int retrieveGrpVersions(const std::string& grpId, std::set<RsNxsGrp*>& grp) = 0;
 
     /*!
      * @param msgId the id of the message to get versions for
      * @param cache whether to store the result in memory
      * @param errCode
      */
-    virtual int retrieveMsgVersions(const std::string& grpId, std::set<RsGxsGroup*> grp, bool cache);
+    virtual int retrieveMsgVersions(const std::string& grpId, const std::string msgId, std::set<RsNxsMsg*> grp) = 0;
 
+    /*!
+     * @param msgId the id of the message to retrieve
+     * @return NULL if message does not exist, or pointer to grp if found
+     */
+    virtual RsNxsMsg* retrieveMsgVersion(const RsGxsMsgId& msgId) = 0;
+
+    /*!
+     * @param grpId the id of the group to retrieve
+     * @return NULL if group does not exist or pointer to grp if found
+     */
+    virtual RsNxsGrp* retrieveGrpVersion(const RsGxsGrpId& grpId) = 0;
 
     /*!
      * allows for more complex queries specific to the service
@@ -117,10 +128,14 @@ public:
     virtual int searchGrps(RsGxsSearch* search, std::list<RsGxsSrchResGrpCtx*>& result) = 0;
 
     /*!
-     * @return the cache size set for this RsGeneralDataService
+     * @return the cache size set for this RsGeneralDataService in bytes
      */
-    virtual uint32_t cacheSize() const;
+    virtual uint32_t cacheSize() const = 0;
 
+    /*!
+     * @param size size of cache to set in bytes
+     */
+    virtual int setCacheSize(uint32_t size) = 0;
 
     /*!
      * Stores a list signed messages into data store
@@ -136,7 +151,6 @@ public:
      */
     virtual int storeGroup(std::set<RsNxsGrp*>& grp) = 0;
 
-
 };
 
 

libretroshare/src/gxs/rsgenexchange.cc

@@ -0,0 +1,5 @@
+#include "rsgenexchange.h"
+
+RsGenExchange::RsGenExchange()
+{
+}

libretroshare/src/gxs/rsgenexchange.h

@@ -0,0 +1,16 @@
+#ifndef RSGENEXCHANGE_H
+#define RSGENEXCHANGE_H
+
+
+
+#include "rsgxs.h"
+
+class RsGenExchange : public RsGxsService
+{
+public:
+    RsGenExchange();
+
+
+};
+
+#endif // RSGENEXCHANGE_H

libretroshare/src/gxs/rsgnp.h

@@ -36,42 +36,6 @@
 #include "gxs/rsgxs.h"
 #include "services/p3service.h"
 
-
-/*!
- * This is universal format for messages that is transported throughout
- * the general exchange service chain. From concrete service to
- */
-class RsGxsSignedMessage : RsItem {
-
-    uint32_t timestamp;
-    void* signature;
-    void* data;
-    uint32_t msg_flags; /* encrypted */
-    std::string msgId; /* hash of all message data */
-    std::string grpId;
-
-};
-
-/*!
- * Item for dealing
- * with grp list
- */
-class RsGxsMessageList : public RsGxsSignedMessage {
-
-};
-
-/*!
- * Item for dealing with group
- * description and msg list
- */
-class RsGxsGroup : public RsGxsSignedMessage {
-
-
-};
-
-
-typedef std::map<std::string, std::set<std::string> > PeerGrp;
-
 /*!
  * Retroshare General Network Exchange Service: \n
  * Interface:

libretroshare/src/gxs/rsgxs.h

@@ -53,6 +53,7 @@ public:
 
     std::string grpId;
     std::string msgId;
+    RsTlvKeySignature idSign;
 };
 
 
@@ -64,9 +65,6 @@ public:
     RsTlvKeySignature adminSign;
 };
 
-typedef time_t RsGxsTime;
-typedef std::map<std::string, uint32_t> IdVersionM;
-typedef std::multimap<std::string, std::pair<std::string, uint32_t> > GrpMsgMap;
 typedef uint64_t    RsGroupId ;
 
 
@@ -162,7 +160,7 @@ public:
      * @return token for request
      * @see receiveMsg()
      */
-    int requestGrpMsgs(const std::list<string>& grpIds);
+    int requestGrpMsgs(const std::list<std::string>& grpIds);
 
     /*!
      * request latest version of messages for group

libretroshare/src/gxs/rsgxsflags.h

@@ -0,0 +1,55 @@
+#ifndef RSGXSFLAGS_H
+#define RSGXSFLAGS_H
+
+#include "inttypes.h"
+
+namespace GXS_SERV {
+
+
+    /*** GROUP FLAGS ***/
+
+    /* type of group */
+
+    static const uint32_t FLAG_GRP_TYPE_MASK;
+
+    // pub key encrypted
+    static const uint32_t FLAG_GRP_TYPE_PRIVATE;
+
+    // single publisher, read only
+    static const uint32_t FLAG_GRP_TYPE_RESTRICTED;
+
+    // anyone can publish
+    static const uint32_t FLAG_GRP_TYPE_PUBLIC;
+
+
+    /* type of msgs allowed */
+
+    static const uint32_t FLAG_MSG_TYPE_MASK;
+
+    // only signee can edit, and sign required
+    static const uint32_t FLAG_MSG_TYPE_SIGNED;
+
+    // no sign required, but signee can edit if signed
+    static const uint32_t FLAG_MSG_TYPE_ANON;
+
+    // anyone can mod but sign must be provided (needed for wikis)
+    static const uint32_t FLAG_MSG_TYPE_SIGNED_SHARED;
+
+    /*** GROUP FLAGS ***/
+
+
+
+    /*** MESSAGE FLAGS ***/
+
+    // indicates message edits an existing message
+    static const uint32_t FLAG_MSG_EDIT;
+
+    // indicates msg is id signed
+    static const uint32_t FLAG_MSG_ID_SIGNED;
+
+    /*** MESSAGE FLAGS ***/
+
+}
+
+
+#endif // RSGXSFLAGS_H

libretroshare/src/libretroshare.pro

@@ -4,7 +4,7 @@ TEMPLATE = lib
 CONFIG += staticlib bitdht newcache# newservices
 CONFIG -= qt
 TARGET = retroshare
-
+#DEFINES += RSSERIAL_DEBUG
 CONFIG += test_voip 
 
 # Beware: All data of the stripped services are lost
@@ -158,7 +158,10 @@ PUBLIC_HEADERS =	retroshare/rsblogs.h \
 
 HEADERS += plugins/pluginmanager.h \
 				plugins/dlfcn_win32.h \
-                                serialiser/rspluginitems.h
+                                serialiser/rspluginitems.h \
+    gxs/rsdataservice.h \
+    gxs/rsgxsflags.h \
+    gxs/rsgenexchange.h
 
 
 HEADERS += $$PUBLIC_HEADERS
@@ -660,21 +663,14 @@ SOURCES +=	zeroconf/p3zeroconf.cc  \
 
 newcache { 
 
-HEADERS += serialiser/rsnxsitems.h
-
-#gxs/rsgxs.h \
-#		gxs/rsgnp.h \
- #   		gxs/rsgdp.h \
-#		util/retrodb.h \
-#		gxs/rsgixs.h
-
-SOURCES += serialiser/rsnxsitems.cpp
-#gxs/rsgxs.cpp \
-#		gxs/rsgnp.cpp \
-#		gxs/rsgdp.cpp \
-#		util/retrodb.cpp \
-#		gxs/rsgixs.cpp
+HEADERS += serialiser/rsnxsitems.h \
+            gxs/rsgds.h \
+            gxs/rsgxs.h \
+            gxs/rsdataservice/h
 
+SOURCES += serialiser/rsnxsitems.cc \
+                gxs/rsdataservice.cc \
+                gxs/rsgenexchange.cc
 }
 
 

libretroshare/src/serialiser/rsgxsitems.h

@@ -6,7 +6,7 @@
  *
  * RetroShare Serialiser.
  *
- * Copyright 2012 Christopher Evi-Parker, Robert Fernie.
+ * Copyright 2012   Christopher Evi-Parker, Robert Fernie.
  *
  * This library is free software; you can redistribute it and/or
  * modify it under the terms of the GNU Library General Public
@@ -39,72 +39,42 @@
 
 /*!
  * Base class used by client API
- * for their messages.
- * Main purpose is to transfer
- * between GDS and client side GXS
+ * for messages.
+ * Main purpose is for msg flows
+ * between GDS and GXS API clients
  */
 class RsGxsMsg : public RsItem
 {
 public:
 
-    RsGxsMsg() : RsItem() {}
+    RsGxsMsg() : RsItem(0) {}
+    virtual ~RsGxsMsg(){ return; }
 
 
-    std::string grpId; /// grp id
-    std::string msgId; /// message id
-    uint32_t timestamp; /// when created
-    std::string identity;
-    uint32_t flag;
-};
-
-/*!
- *
- *
- *
- */
-class RsGxsSignedMsg : public RsItem
-{
-
-    RsGxsSignedMsg(uint16_t servtype, uint8_t subtype)
-        : RsItem(RS_PKT_VERSION_SERVICE, servtype, subtype) { return; }
-
-    virtual ~RsGxsSignedMsg() { return; }
-
-    virtual void clear();
-    virtual std::ostream& print(std::ostream &out, uint16_t indent = 0);
+    /***** Don not serialise or set these members *****/
 
     std::string grpId; /// group id
     std::string msgId; /// message id
-    uint32_t timestamp;
-    std::string identity;
-    uint32_t flag;
-    RsTlvBinaryData* msg;
-    RsTlvBinaryData signature; // needs to be a set of signatures with bitfield flag ids
-    // set should cover idenitity and publication key
+    uint32_t timeStamp; /// UTC time when created
+    std::string identity; /// identity associated with (no id means no signed)
+
+    /*!
+     * The purpose of the flag is to note
+     * If this message is an edit (note that edits only applies to
+     * signed msgs)
+     */
+    uint32_t msgFlag;
+
+    /***** Don not serialise or set these members *****/
+
 };
 
-/*!
- *
- *
- *
- *
- */
-class RsGxsSignedGroup
-{
-
-
-    std::string grpId;
-    std::string identity;
-    RsTlvKeySignature sign;
-    uint32_t timestamp; // UTC time
-    RsTlvBinaryData* grp;
-    uint32_t flag;
-};
 
 /*!
- * This is the base group used by API client to exchange
- * data
- *
+ * Base class used by client API
+ * for groups.
+ * Main purpose is for msg flows
+ * between GDS and GXS API clients
  */
 class RsGxsGroup : public RsItem
 {
@@ -112,47 +82,67 @@ class RsGxsGroup : public RsItem
 
 public:
 
-    // publication type
-    static const uint32_t FLAG_GRP_TYPE_KEY_SHARED;
-    static const uint32_t FLAG_GRP_TYPE_PRIVATE_ENC;
-    static const uint32_t FLAG_GRP_TYPE_PUBLIC;
-    static const uint32_t FLAG_GRP_TYPE_MASK;
 
-    // type of msgs
-    static const uint32_t FLAG_MSGS_TYPE_SIGNED;
-    static const uint32_t FLAG_MSGS_TYPE_ANON;
-    static const uint32_t FLAG_MSGS_TYPE_MASK;
+
+    /*** type of msgs ***/
 
     RsGxsGroup(uint16_t servtype, uint8_t subtype)
         : RsItem(servtype) { return; }
 
     virtual ~RsGxsGroup() { return; }
 
-    std::string grpId;
+
+    std::string grpId; /// group id
     std::string identity; /// identity associated with group
-    uint32_t timeStamp; // UTC time
-    uint32_t flag;
+    uint32_t timeStamp; /// UTC time
+    bool subscribed;
+    bool read;
+
+    /*!
+     * Three thing flag represents:
+     * Is it signed by identity?
+     * Group type (private, public, restricted)
+     * msgs allowed (signed and anon)
+     */
+    uint32_t grpFlag;
+
+    /*****                          *****/
 
 
 };
 
-class RsGxsKey : public RsItem {
-
-
 
+class RsGxsSearch : public RsItem {
 
+    RsGxsSearch(uint32_t servtype) : RsItem(servtype) { return ; }
+    virtual ~RsGxsSearch() { return;}
 
 };
 
+class RsGxsSrchResMsgCtx : public RsItem {
+
+    RsGxsSrchResMsgCtx(uint32_t servtype) : RsItem(servtype) { return; }
+    virtual ~RsGxsSrchResMsgCtx() {return; }
+    std::string msgId;
+    RsTlvKeySignature sign;
+
+};
+
+class RsGxsSrchResGrpCtx : public RsItem {
+
+    RsGxsSrchResGrpCtx(uint32_t servtype) : RsItem(servtype) { return; }
+    virtual ~RsGxsSrchResGrpCtx() {return; }
+    std::string msgId;
+    RsTlvKeySignature sign;
+
+};
 
 class RsGxsSerialiser : public RsSerialType
 {
-
 public:
 
-
-
-
+    RsGxsSerialiser(uint32_t servtype) : RsSerialType(servtype) { return; }
+    virtual ~RsGxsSerialiser() { return; }
 };
 
 

libretroshare/src/serialiser/rsnxsitems.cc

@@ -15,8 +15,8 @@ const uint8_t RsSyncGrpMsg::FLAG_USE_SYNC_HASH = 0x001;
 
 uint32_t RsNxsSerialiser::size(RsItem *item) {
 
-    RsGrpResp* grp;
-    RsGrpMsgResp* gmp;
+    RsNxsGrp* ngp;
+    RsNxsMsg* nmg;
     RsSyncGrp* sg;
     RsSyncGrpList* sgl;
     RsSyncGrpMsg* sgm;
@@ -37,12 +37,12 @@ uint32_t RsNxsSerialiser::size(RsItem *item) {
     }else if ((sgml = dynamic_cast<RsSyncGrpMsgList*>(item)) != NULL)
     {
         sizeSyncGrpMsgList(sgml);
-    }else if((grp = dynamic_cast<RsGrpResp*>(item)) != NULL)
+    }else if((ngp = dynamic_cast<RsNxsGrp*>(item)) != NULL)
     {
-        sizeGrpResp(grp);
-    }else if((gmp = dynamic_cast<RsGrpMsgResp*>(item)) != NULL)
+        sizeNxsGrp(ngp);
+    }else if((nmg = dynamic_cast<RsNxsMsg*>(item)) != NULL)
     {
-        sizeGrpMsgResp(gmp);
+        sizeNxsMsg(nmg);
     }
 }
 
@@ -73,18 +73,16 @@ RsItem* RsNxsSerialiser::deserialise(void *data, uint32_t *size) {
             return deserialSyncGrpMsg(data, size);
         case RS_PKT_SUBTYPE_SYNC_MSG_LIST:
             return deserialSyncGrpMsgList(data, size);
-        case RS_PKT_SUBTYPE_GRPS_RESP:
-            return deserialGrpResp(data, size);
-        case RS_PKT_SUBTYPE_MSG_RESP:
-            return deserialGrpMsgResp(data, size);
-        case RS_PKT_SUBTYPE_SEARCH_REQ:
-            return deserialNxsSearchReq(data, size);
-        case RS_PKT_SUBTYPE_SEARCH_RESP:
-            return deserialNxsSearchResp(data, size);
+        case RS_PKT_SUBTYPE_NXS_GRP:
+            return deserialNxsGrp(data, size);
+        case RS_PKT_SUBTYPE_NXS_MSG:
+            return deserialNxsMsg(data, size);
+        case RS_PKT_SUBTYPE_NXS_EXTENDED:
+            return deserialNxsExtended(data, size);
         default:
             {
 #ifdef NXS_DEBUG
-                std::cerr << "RsNxsSerialiser::deserialise() : data has not type"
+                std::cerr << "RsNxsSerialiser::deserialise() : data has no type"
                           << std::endl;
 #endif
                 return NULL;
@@ -97,12 +95,13 @@ RsItem* RsNxsSerialiser::deserialise(void *data, uint32_t *size) {
 
 bool RsNxsSerialiser::serialise(RsItem *item, void *data, uint32_t *size){
 
-    RsGrpResp* grp;
-    RsGrpMsgResp* gmp;
+    RsNxsGrp* ngp;
+    RsNxsMsg* nmg;
     RsSyncGrp* sg;
     RsSyncGrpList* sgl;
     RsSyncGrpMsg* sgm;
     RsSyncGrpMsgList* sgml;
+    RsNxsExtended* nxt;
 
     if((sg = dynamic_cast<RsSyncGrp*>(item))  != NULL)
     {
@@ -118,14 +117,16 @@ bool RsNxsSerialiser::serialise(RsItem *item, void *data, uint32_t *size){
     }else if ((sgml = dynamic_cast<RsSyncGrpMsgList*>(item)) != NULL)
     {
         return serialiseSynGrpMsgList(sgml, data, size);
-    }else if((grp = dynamic_cast<RsGrpResp*>(item)) != NULL)
+    }else if((ngp = dynamic_cast<RsNxsGrp*>(item)) != NULL)
     {
-        return serialiseGrpResp(grp, data, size);
-    }else if((gmp = dynamic_cast<RsGrpMsgResp*>(item)) != NULL)
+        return serialiseNxsGrp(ngp, data, size);
+    }else if((nmg = dynamic_cast<RsNxsMsg*>(item)) != NULL)
     {
-        return serialiseGrpMsgResp(gmp, data, size);
-    }
+        return serialiseNxsMsg(nmg, data, size);
+    }else if((nxt = dynamic_cast<RsNxsExtended*>(item)) != NULL){
 
+        return serialiseNxsExtended(nxt, data, size);
+    }
 
 #ifdef NXS_DEBUG
     std::cerr << "RsNxsSerialiser::serialise() item does not caste to know type"
@@ -166,7 +167,7 @@ bool RsNxsSerialiser::serialiseSynGrpMsgList(RsSyncGrpMsgList *item, void *data,
     ok &= setRawUInt8(data, *size, &offset, item->flag);
     ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId);
 
-    std::map<std::string, std::list<uint32_t> >::iterator mit =
+    SyncList::iterator mit =
             item->msgs.begin();
 
     for(; mit != item->msgs.end(); mit++){
@@ -179,10 +180,11 @@ bool RsNxsSerialiser::serialiseSynGrpMsgList(RsSyncGrpMsgList *item, void *data,
 
         ok &= setRawUInt32(data, *size, &offset, nVersions);
 
-        std::list<uint32_t>::iterator lit = mit->second.begin();
+        std::list<RsTlvKeySignature>::iterator lit = mit->second.begin();
         for(; lit != mit->second.end(); lit++)
         {
-            ok &= SetTlvUInt32(data, *size, &offset, TLV_TYPE_UINT32_AGE, *lit);
+            RsTlvKeySignature& b = *lit;
+            b.SetTlv(data, *size, &offset);
         }
     }
 
@@ -204,19 +206,19 @@ bool RsNxsSerialiser::serialiseSynGrpMsgList(RsSyncGrpMsgList *item, void *data,
 }
 
 
-bool RsNxsSerialiser::serialiseGrpMsgResp(RsGrpMsgResp *item, void *data, uint32_t *size)
+bool RsNxsSerialiser::serialiseNxsMsg(RsNxsMsg *item, void *data, uint32_t *size)
 {
 
 #ifdef RSSERIAL_DEBUG
-    std::cerr << "RsNxsSerialiser::serialiseGrpMsgResp()" << std::endl;
+    std::cerr << "RsNxsSerialiser::serialiseNxsMsg()" << std::endl;
 #endif
 
-    uint32_t tlvsize = sizeGrpMsgResp(item);
+    uint32_t tlvsize = sizeNxsMsg(item);
     uint32_t offset = 0;
 
     if(*size < tlvsize){
 #ifdef RSSERIAL_DEBUG
-        std::cerr << "RsNxsSerialiser::serialiseGrpMsgResp()" << std::endl;
+        std::cerr << "RsNxsSerialiser::serialiseNxsMsg()" << std::endl;
 #endif
         return false;
     }
@@ -230,18 +232,20 @@ bool RsNxsSerialiser::serialiseGrpMsgResp(RsGrpMsgResp *item, void *data, uint32
     /* skip the header */
     offset += 8;
 
-    /* RsSyncGrpMsgList */
 
-    std::list<RsTlvBinaryData*>::iterator lit = item->msgs.begin();
+    ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_MSGID, item->msgId);
+    ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_GROUPID, item->grpId);
+    ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_NAME, item->identity);
+    ok &= setRawUInt32(data, tlvsize, &offset, item->timeStamp);
+    ok &= setRawUInt32(data, tlvsize, &offset, item->msgFlag);
+    ok &= item->idSign.SetTlv(data, tlvsize, &offset);
+    ok &= item->publishSign.SetTlv(data, tlvsize, &offset);
+    ok &= item->msg.SetTlv(data, tlvsize, &offset);
 
-    for(; lit != item->msgs.end(); lit++){
-        RsTlvBinaryData*& b = *lit;
-        b->SetTlv(data, *size, &offset);
-    }
 
     if(offset != tlvsize){
 #ifdef RSSERIAL_DEBUG
-        std::cerr << "RsNxsSerialiser::serialiseGrpMsgResp() FAIL Size Error! " << std::endl;
+        std::cerr << "RsNxsSerialiser::serialiseNxsMsg() FAIL Size Error! " << std::endl;
 #endif
         ok = false;
     }
@@ -249,7 +253,7 @@ bool RsNxsSerialiser::serialiseGrpMsgResp(RsGrpMsgResp *item, void *data, uint32
 #ifdef RSSERIAL_DEBUG
     if (!ok)
     {
-        std::cerr << "RsNxsSerialiser::serialiseGrpMsgResp() NOK" << std::endl;
+        std::cerr << "RsNxsSerialiser::serialiseNxsMsg() NOK" << std::endl;
     }
 #endif
 
@@ -257,18 +261,18 @@ bool RsNxsSerialiser::serialiseGrpMsgResp(RsGrpMsgResp *item, void *data, uint32
 }
 
 
-bool RsNxsSerialiser::serialiseGrpResp(RsGrpResp *item, void *data, uint32_t *size){
+bool RsNxsSerialiser::serialiseNxsGrp(RsNxsGrp *item, void *data, uint32_t *size){
 
 #ifdef RSSERIAL_DEBUG
-    std::cerr << "RsNxsSerialiser::serialiseGrpResp()" << std::endl;
+    std::cerr << "RsNxsSerialiser::serialiseNxsGrp()" << std::endl;
 #endif
 
-    uint32_t tlvsize = sizeGrpResp(item);
+    uint32_t tlvsize = sizeNxsGrp(item);
     uint32_t offset = 0;
 
     if(*size < tlvsize){
 #ifdef RSSERIAL_DEBUG
-        std::cerr << "RsNxsSerialiser::serialiseGrpResp()" << std::endl;
+        std::cerr << "RsNxsSerialiser::serialiseNxsGrp()" << std::endl;
 #endif
         return false;
     }
@@ -282,16 +286,19 @@ bool RsNxsSerialiser::serialiseGrpResp(RsGrpResp *item, void *data, uint32_t *si
     /* skip the header */
     offset += 8;
 
-    std::list<RsTlvBinaryData*>::iterator lit = item->grps.begin();
+    // grp id
 
-    for(; lit != item->grps.end(); lit++){
-        RsTlvBinaryData*& b = *lit;
-        b->SetTlv(data, *size, &offset);
-    }
+    ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_GROUPID, item->grpId);
+    ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_NAME, item->identity);
+    ok &= setRawUInt32(data, tlvsize, &offset, item->grpFlag);
+    ok &= setRawUInt32(data, tlvsize, &offset, item->timeStamp);
+    ok &= item->idSign.SetTlv(data, tlvsize, &offset);
+    ok &= item->adminSign.SetTlv(data, tlvsize, &offset);
+    ok &= item->keys.SetTlv(data, tlvsize, &offset);
 
     if(offset != tlvsize){
 #ifdef RSSERIAL_DEBUG
-        std::cerr << "RsNxsSerialiser::serialiseGrpResp() FAIL Size Error! " << std::endl;
+        std::cerr << "RsNxsSerialiser::serialiseNxsGrp() FAIL Size Error! " << std::endl;
 #endif
         ok = false;
     }
@@ -299,7 +306,7 @@ bool RsNxsSerialiser::serialiseGrpResp(RsGrpResp *item, void *data, uint32_t *si
 #ifdef RSSERIAL_DEBUG
     if (!ok)
     {
-        std::cerr << "RsNxsSerialiser::serialiseGrpResp() NOK" << std::endl;
+        std::cerr << "RsNxsSerialiser::serialiseNxsGrp() NOK" << std::endl;
     }
 #endif
 
@@ -381,7 +388,7 @@ bool RsNxsSerialiser::serialiseSyncGrpList(RsSyncGrpList *item, void *data, uint
 
     /* RsSyncGrpList */
 
-    std::map<std::string, std::list<uint32_t> >::iterator mit =
+    SyncList::iterator mit =
             item->grps.begin();
 
 
@@ -397,10 +404,11 @@ bool RsNxsSerialiser::serialiseSyncGrpList(RsSyncGrpList *item, void *data, uint
 
         ok &= setRawUInt32(data, *size, &offset, nVersions);
 
-        std::list<uint32_t>::iterator lit = mit->second.begin();
+        std::list<RsTlvKeySignature>::iterator lit = mit->second.begin();
         for(; lit != mit->second.end(); lit++)
         {
-            ok &= SetTlvUInt32(data, *size, &offset, TLV_TYPE_UINT32_AGE, *lit);
+            RsTlvKeySignature& b = *lit;
+            b.SetTlv(data, *size, &offset);
         }
     }
 
@@ -473,15 +481,16 @@ bool RsNxsSerialiser::serialiseNxsSearchReq(RsNxsSearchReq *item, void *data, ui
     return false;
 }
 
-bool RsNxsSerialiser::serialiseNxsSearchResp(RsNxsSearchResp *item, void *data, uint32_t *size){
+
+bool RsNxsSerialiser::serialiseNxsExtended(RsNxsExtended *item, void *data, uint32_t *size){
+
     return false;
 }
 
-
 /*** deserialisation ***/
 
 
-RsGrpResp* RsNxsSerialiser::deserialGrpResp(void *data, uint32_t *size){
+RsNxsGrp* RsNxsSerialiser::deserialNxsGrp(void *data, uint32_t *size){
 
 #ifdef RSSERIAL_DEBUG
     std::cerr << "RsNxsSerialiser::deserialGrpResp()" << std::endl;
@@ -494,10 +503,10 @@ RsGrpResp* RsNxsSerialiser::deserialGrpResp(void *data, uint32_t *size){
 
     if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) ||
             (SERVICE_TYPE != getRsItemService(rstype)) ||
-            (RS_PKT_SUBTYPE_GRPS_RESP != getRsItemSubType(rstype)))
+            (RS_PKT_SUBTYPE_NXS_GRP != getRsItemSubType(rstype)))
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpResp() FAIL wrong type" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsGrp() FAIL wrong type" << std::endl;
 #endif
             return NULL; /* wrong type */
     }
@@ -505,7 +514,7 @@ RsGrpResp* RsNxsSerialiser::deserialGrpResp(void *data, uint32_t *size){
     if (*size < rssize)    /* check size */
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpResp() FAIL wrong size" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsGrp() FAIL wrong size" << std::endl;
 #endif
             return NULL; /* not enough data */
     }
@@ -515,32 +524,23 @@ RsGrpResp* RsNxsSerialiser::deserialGrpResp(void *data, uint32_t *size){
 
     bool ok = true;
 
-    RsGrpResp* item = new RsGrpResp(SERVICE_TYPE);
+    RsNxsGrp* item = new RsNxsGrp(SERVICE_TYPE);
 
     /* skip the header */
     offset += 8;
 
-    while(offset < *size && ok){
-
-        RsTlvBinaryData *b = new RsTlvBinaryData(SERVICE_TYPE);
-        ok &= b->GetTlv(data, *size, &offset);
-
-        if(ok)
-            item->grps.push_back(b);
-        else{
-            delete b;
-
-#ifdef RSSERIAL_DEBUG
-            std::cerr <<  "RsNxsSerialiser::deserialGrpResp(): Failure to deserialise binary group!" << std::endl;
-#endif
-        }
-    }
-
+    ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId);
+    ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_NAME, item->identity);
+    ok &= getRawUInt32(data, *size, &offset, &(item->grpFlag));
+    ok &= getRawUInt32(data, *size, &offset, &(item->timeStamp));
+    ok &= item->idSign.GetTlv(data, *size, &offset);
+    ok &= item->adminSign.GetTlv(data, *size, &offset);
+    ok &= item->keys.GetTlv(data, *size, &offset);
 
     if (offset != rssize)
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpResp() FAIL size mismatch" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl;
 #endif
             /* error */
             delete item;
@@ -550,7 +550,7 @@ RsGrpResp* RsNxsSerialiser::deserialGrpResp(void *data, uint32_t *size){
     if (!ok)
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpResp() NOK" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsGrp() NOK" << std::endl;
 #endif
             delete item;
             return NULL;
@@ -560,10 +560,10 @@ RsGrpResp* RsNxsSerialiser::deserialGrpResp(void *data, uint32_t *size){
 }
 
 
-RsGrpMsgResp* RsNxsSerialiser::deserialGrpMsgResp(void *data, uint32_t *size){
+RsNxsMsg* RsNxsSerialiser::deserialNxsMsg(void *data, uint32_t *size){
 
 #ifdef RSSERIAL_DEBUG
-    std::cerr << "RsNxsSerialiser::deserialGrpResp()" << std::endl;
+    std::cerr << "RsNxsSerialiser::deserialNxsMsg()" << std::endl;
 #endif
     /* get the type and size */
     uint32_t rstype = getRsItemId(data);
@@ -574,10 +574,10 @@ RsGrpMsgResp* RsNxsSerialiser::deserialGrpMsgResp(void *data, uint32_t *size){
 
     if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) ||
             (SERVICE_TYPE != getRsItemService(rstype)) ||
-            (RS_PKT_SUBTYPE_MSG_RESP != getRsItemSubType(rstype)))
+            (RS_PKT_SUBTYPE_NXS_MSG != getRsItemSubType(rstype)))
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpMsgResp() FAIL wrong type" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsMsg() FAIL wrong type" << std::endl;
 #endif
             return NULL; /* wrong type */
     }
@@ -585,7 +585,7 @@ RsGrpMsgResp* RsNxsSerialiser::deserialGrpMsgResp(void *data, uint32_t *size){
     if (*size < rssize)    /* check size */
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpMsgResp() FAIL wrong size" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsMsg() FAIL wrong size" << std::endl;
 #endif
             return NULL; /* not enough data */
     }
@@ -595,30 +595,24 @@ RsGrpMsgResp* RsNxsSerialiser::deserialGrpMsgResp(void *data, uint32_t *size){
 
     bool ok = true;
 
-    RsGrpMsgResp* item = new RsGrpMsgResp(getRsItemService(rstype));
+    RsNxsMsg* item = new RsNxsMsg(getRsItemService(rstype));
     /* skip the header */
+
     offset += 8;
 
-    while(offset < *size){
-
-        RsTlvBinaryData *b = new RsTlvBinaryData(SERVICE_TYPE);
-        ok &= b->GetTlv(data, *size, &offset);
-
-        if(ok)
-            item->msgs.push_back(b);
-        else
-        {
-            delete b;
-#ifdef RSSERIAL_DEBUG
-            std::cerr <<  "RsNxsSerialiser::deserialGrpMsgResp(): Failure to deserialise binary message!" << std::endl;
-#endif
-        }
-    }
+    ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_MSGID, item->msgId);
+    ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId);
+    ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_NAME, item->identity);
+    ok &= getRawUInt32(data, *size, &offset, &(item->timeStamp));
+    ok &= getRawUInt32(data, *size, &offset, &(item->msgFlag));
+    ok &= item->idSign.GetTlv(data, *size, &offset);
+    ok &= item->publishSign.GetTlv(data, *size, &offset);
+    ok &= item->msg.GetTlv(data, *size, &offset);;
 
     if (offset != rssize)
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpMsgResp() FAIL size mismatch" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsMsg() FAIL size mismatch" << std::endl;
 #endif
             /* error */
             delete item;
@@ -628,7 +622,7 @@ RsGrpMsgResp* RsNxsSerialiser::deserialGrpMsgResp(void *data, uint32_t *size){
     if (!ok)
     {
 #ifdef RSSERIAL_DEBUG
-            std::cerr << "RsNxsSerialiser::deserialGrpMsgResp() NOK" << std::endl;
+            std::cerr << "RsNxsSerialiser::deserialNxsMsg() NOK" << std::endl;
 #endif
             delete item;
             return NULL;
@@ -753,17 +747,14 @@ RsSyncGrpList* RsNxsSerialiser::deserialSyncGrpList(void *data, uint32_t *size){
         uint32_t nVersions;
         ok &= getRawUInt32(data, *size, &offset, &nVersions);
 
-        std::list<uint32_t> verL;
         uint32_t verCount  = 0;
         while((offset < rssize) && ok && (verCount < nVersions))
         {
-            uint32_t version;
-            ok &= GetTlvUInt32(data, *size, &offset, TLV_TYPE_UINT32_AGE, &version);
-            verL.push_back(version);
+            RsTlvKeySignature sign;
+            ok &= sign.GetTlv(data, *size, &offset);
+            item->grps[grpId].push_back(sign);
             verCount++;
         }
-
-        item->grps[grpId] = verL;
     }
 
     if (offset != rssize)
@@ -841,17 +832,15 @@ RsSyncGrpMsgList* RsNxsSerialiser::deserialSyncGrpMsgList(void *data, uint32_t *
         uint32_t nVersions;
         ok &= getRawUInt32(data, *size, &offset, &nVersions);
 
-        std::list<uint32_t> verL;
         uint32_t verCount  = 0;
         while((offset < rssize) && ok && (verCount < nVersions))
         {
-            uint32_t version;
-            ok &= GetTlvUInt32(data, *size, &offset, TLV_TYPE_UINT32_AGE, &version);
-            verL.push_back(version);
+            RsTlvKeySignature sign;
+            ok &= sign.GetTlv(data, *size, &offset);
+            item->msgs[msgId].push_back(sign);
             verCount++;
         }
 
-        item->msgs[msgId] = verL;
     }
 
     if (offset != rssize)
@@ -952,9 +941,7 @@ RsNxsSearchReq* RsNxsSerialiser::deserialNxsSearchReq(void *data, uint32_t *size
     return NULL;
 }
 
-RsNxsSearchResp* RsNxsSerialiser::deserialNxsSearchResp(void *data, uint32_t *size)
-{
-
+RsNxsExtended* RsNxsSerialiser::deserialNxsExtended(void *data, uint32_t *size){
     return NULL;
 }
 
@@ -963,31 +950,35 @@ RsNxsSearchResp* RsNxsSerialiser::deserialNxsSearchResp(void *data, uint32_t *si
 /*** size functions ***/
 
 
-uint32_t RsNxsSerialiser::sizeGrpMsgResp(RsGrpMsgResp *item)
+uint32_t RsNxsSerialiser::sizeNxsMsg(RsNxsMsg *item)
 {
 
     uint32_t s = 8; //header size
 
-    std::list<RsTlvBinaryData*>::iterator it =
-            item->msgs.begin();
-
-    for(; it != item->msgs.end(); it++)
-        s += (*it)->TlvSize();
+    s += GetTlvStringSize(item->grpId);
+    s += GetTlvStringSize(item->msgId);
+    s += GetTlvStringSize(item->identity);
+    s += 4; // msgFlag
+    s += 4; // timeStamp
+    s += item->publishSign.TlvSize();
+    s += item->idSign.TlvSize();
+    s += item->msg.TlvSize();
 
     return s;
 }
 
-uint32_t RsNxsSerialiser::sizeGrpResp(RsGrpResp *item)
+uint32_t RsNxsSerialiser::sizeNxsGrp(RsNxsGrp *item)
 {
-
     uint32_t s = 8; // header size
 
-    std::list<RsTlvBinaryData*>::iterator it =
-            item->grps.begin();
-
-    for(; it != item->grps.end(); it++)
-        s += (*it)->TlvSize();
-
+    s += GetTlvStringSize(item->grpId);
+    s += GetTlvStringSize(item->identity);
+    s += 4; // grpFlag
+    s += 4; // timestamp
+    s += item->adminSign.TlvSize();
+    s += item->idSign.TlvSize();
+    s += item->keys.TlvSize();
+    s += item->grp.TlvSize();
 
     return s;
 }
@@ -1011,7 +1002,7 @@ uint32_t RsNxsSerialiser::sizeSyncGrpList(RsSyncGrpList *item)
 
     s += 1; // flag
 
-    std::map<std::string, std::list<uint32_t> >::iterator mit
+    std::map<std::string, std::list<RsTlvKeySignature> >::iterator mit
             = item->grps.begin();
 
     for(; mit != item->grps.end(); mit++){
@@ -1019,12 +1010,13 @@ uint32_t RsNxsSerialiser::sizeSyncGrpList(RsSyncGrpList *item)
         s += 4; // number of versions
         s += GetTlvStringSize(mit->first);
 
-        const std::list<uint32_t>& verL = mit->second;
-        std::list<uint32_t>::const_iterator lit =
+        std::list<RsTlvKeySignature>& verL = mit->second;
+        std::list<RsTlvKeySignature>::iterator lit =
                 verL.begin();
 
         for(; lit != verL.end(); lit++){
-            s += GetTlvUInt32Size(); // version
+            RsTlvKeySignature& sign = *lit; // version
+            s += sign.TlvSize();
         }
     }
     return s;
@@ -1051,7 +1043,7 @@ uint32_t RsNxsSerialiser::sizeSyncGrpMsgList(RsSyncGrpMsgList *item)
 
     s += 1; // flag
     s += GetTlvStringSize(item->grpId);
-    std::map<std::string, std::list<uint32_t> >::iterator mit
+    std::map<std::string, std::list<RsTlvKeySignature> >::iterator mit
             = item->msgs.begin();
 
     for(; mit != item->msgs.end(); mit++){
@@ -1059,52 +1051,47 @@ uint32_t RsNxsSerialiser::sizeSyncGrpMsgList(RsSyncGrpMsgList *item)
         s += 4; // number of versions
         s += GetTlvStringSize(mit->first);
 
-        const std::list<uint32_t>& verL = mit->second;
-        std::list<uint32_t>::const_iterator lit =
+        std::list<RsTlvKeySignature>& verL = mit->second;
+        std::list<RsTlvKeySignature>::iterator lit =
                 verL.begin();
 
         for(; lit != verL.end(); lit++){
-            s += GetTlvUInt32Size(); // version
+            RsTlvKeySignature& sign = *lit; // version
+            s += sign.TlvSize();
         }
     }
     return s;
 }
 
-
-uint32_t RsNxsSerialiser::sizeNxsSearchReq(RsNxsSearchReq *item)
-{
-    return 0;
-}
-
-uint32_t RsNxsSerialiser::sizeNxsSearchResp(RsNxsSearchResp *item)
-{
+uint32_t RsNxsSerialiser::sizeNxsSearchReq(RsNxsSearchReq *item){
     return 0;
 }
 
 
 /** print and clear functions **/
 
-void RsGrpMsgResp::clear()
+void RsNxsMsg::clear()
 {
-
-    std::list<RsTlvBinaryData*>::iterator it =
-            msgs.begin();
-
-    for(; it != msgs.end(); it++)
-        (*it)->TlvClear();
-
-    msgs.clear();
+    msg.TlvClear();
+    grpId.clear();
+    msgId.clear();
+    msgFlag = 0;
+    timeStamp = 0;
+    publishSign.TlvClear();
+    idSign.TlvClear();
+    identity.clear();
 }
 
-void RsGrpResp::clear()
+void RsNxsGrp::clear()
 {
-    std::list<RsTlvBinaryData*>::iterator it =
-            grps.begin();
-
-    for(; it != grps.end(); it++)
-        (*it)->TlvClear();
-
-    grps.clear();
+    grpId.clear();
+    timeStamp = 0;
+    grp.TlvClear();
+    adminSign.TlvClear();
+    keys.TlvClear();
+    identity.clear();
+    grpFlag = 0;
+    idSign.TlvClear();
 }
 
 void RsSyncGrp::clear()
@@ -1153,6 +1140,20 @@ std::ostream& RsSyncGrp::print(std::ostream &out, uint16_t indent)
     return out;
 }
 
+std::ostream& RsNxsExtended::print(std::ostream &out, uint16_t indent){
+    printRsItemBase(out, "RsNxsExtended", indent);
+    uint16_t int_Indent = indent + 2;
+
+    printIndent(out , int_Indent);
+    out << "type: " << type << std::endl;
+    printIndent(out , int_Indent);
+    extData.print(out, indent);
+
+    printRsItemEnd(out ,"RsNxsExtended", indent);
+
+    return out;
+}
+
 std::ostream& RsSyncGrpMsg::print(std::ostream &out, uint16_t indent)
 {
     printRsItemBase(out, "RsSyncGrpMsg", indent);
@@ -1180,7 +1181,7 @@ std::ostream& RsSyncGrpList::print(std::ostream &out, uint16_t indent)
     out << "flag: " << flag << std::endl;
     printIndent(out , int_Indent);
 
-    std::map<std::string, std::list<uint32_t> >::iterator mit
+    SyncList::iterator mit
             = grps.begin();
 
     for(; mit != grps.end(); mit++){
@@ -1188,13 +1189,13 @@ std::ostream& RsSyncGrpList::print(std::ostream &out, uint16_t indent)
         out << "grpId: " <<  mit->first;
         printIndent(out , int_Indent);
 
-        const std::list<uint32_t>& verL = mit->second;
-        std::list<uint32_t>::const_iterator lit =
+        std::list<RsTlvKeySignature>& verL = mit->second;
+        std::list<RsTlvKeySignature>::iterator lit =
                 verL.begin();
 
         for(; lit != verL.end(); lit++){
-            out << "version: " << *lit << std::endl;
-            printIndent(out , int_Indent);
+            RsTlvKeySignature& sign = *lit;
+            sign.print(out, indent);
         }
     }
 
@@ -1202,6 +1203,8 @@ std::ostream& RsSyncGrpList::print(std::ostream &out, uint16_t indent)
     return out;
 }
 
+
+
 std::ostream& RsSyncGrpMsgList::print(std::ostream &out, uint16_t indent)
 {
     printRsItemBase(out, "RsSyncGrpMsgList", indent);
@@ -1212,7 +1215,7 @@ std::ostream& RsSyncGrpMsgList::print(std::ostream &out, uint16_t indent)
     printIndent(out , int_Indent);
     out << "grpId: " << grpId << std::endl;
 
-    std::map<std::string, std::list<uint32_t> >::iterator mit
+    SyncList::iterator mit
             = msgs.begin();
 
     for(; mit != msgs.end(); mit++){
@@ -1220,13 +1223,13 @@ std::ostream& RsSyncGrpMsgList::print(std::ostream &out, uint16_t indent)
         out << "msgId: " <<  mit->first;
         printIndent(out , int_Indent);
 
-        const std::list<uint32_t>& verL = mit->second;
-        std::list<uint32_t>::const_iterator lit =
+        std::list<RsTlvKeySignature>& verL = mit->second;
+        std::list<RsTlvKeySignature>::iterator lit =
                 verL.begin();
 
         for(; lit != verL.end(); lit++){
-            out << "version: " << *lit << std::endl;
-            printIndent(out , int_Indent);
+           RsTlvKeySignature& sign = *lit;
+           sign.print(out, indent);
         }
     }
 
@@ -1234,32 +1237,49 @@ std::ostream& RsSyncGrpMsgList::print(std::ostream &out, uint16_t indent)
     return out;
 }
 
-std::ostream& RsGrpMsgResp::print(std::ostream &out, uint16_t indent){
+std::ostream& RsNxsGrp::print(std::ostream &out, uint16_t indent){
 
-    printRsItemBase(out, "RsGrpMsgResp", indent);
+    printRsItemBase(out, "RsNxsGrp", indent);
+    uint16_t int_Indent = indent + 2;
 
-    std::list<RsTlvBinaryData*>::iterator it =
-            msgs.begin();
+    out << "grpId: " << grpId << std::endl;
+    printIndent(out , int_Indent);
+    out << "timeStamp: " << timeStamp << std::endl;
+    printIndent(out , int_Indent);
+    out << "identity: " << identity << std::endl;
+    printIndent(out , int_Indent);
+    out << "grpFlag: " << grpFlag << std::endl;
+    out << "adminSign: " << std::endl;
+    adminSign.print(out, indent);
+    out << "idSign: " << std::endl;
+    idSign.print(out, indent);
+    out << "keys: " << std::endl;
+    keys.print(out, indent);
 
-    for(; it != msgs.end(); it++)
-        (*it)->print(out, indent);
-
-
-    printRsItemEnd(out, "RsGrpMsgResp", indent);
+    printRsItemEnd(out, "RsNxsGrp", indent);
     return out;
 }
 
-std::ostream& RsGrpResp::print(std::ostream &out, uint16_t indent){
+std::ostream& RsNxsMsg::print(std::ostream &out, uint16_t indent){
 
-    printRsItemBase(out, "RsGrpResp", indent);
+    printRsItemBase(out, "RsNxsMsg", indent);
+    uint16_t int_Indent = indent + 2;
 
-    std::list<RsTlvBinaryData*>::iterator it =
-            grps.begin();
+    out << "msgId: " << msgId << std::endl;
+    printIndent(out , int_Indent);
+    out << "grpId: " << grpId << std::endl;
+    printIndent(out , int_Indent);
+    out << "msgFlag: " << msgFlag << std::endl;
+    printIndent(out , int_Indent);
+    out << "identity: " << identity << std::endl;
+    printIndent(out , int_Indent);
+    out << "timeStamp: " << timeStamp << std::endl;
+    printIndent(out , int_Indent);
+    out << "pub sign: " << std::endl;
+    publishSign.print(out, indent);
+    out << "id sign: " << std::endl;
+    idSign.print(out, indent);
 
-    for(; it != grps.end(); it++)
-        (*it)->print(out, indent);
-
-
-    printRsItemEnd(out ,"RsGrpMsgResp", indent);
+    printRsItemEnd(out ,"RsNxsMsg", indent);
     return out;
 }

libretroshare/src/serialiser/rsnxsitems.h

@@ -38,12 +38,19 @@
 
 const uint8_t RS_PKT_SUBTYPE_SYNC_GRP      = 0x0001;
 const uint8_t RS_PKT_SUBTYPE_SYNC_GRP_LIST = 0x0002;
-const uint8_t RS_PKT_SUBTYPE_GRPS_RESP     = 0x0004;
+const uint8_t RS_PKT_SUBTYPE_NXS_GRP     = 0x0004;
 const uint8_t RS_PKT_SUBTYPE_SYNC_MSG      = 0x0008;
 const uint8_t RS_PKT_SUBTYPE_SYNC_MSG_LIST = 0x0010;
-const uint8_t RS_PKT_SUBTYPE_MSG_RESP      = 0x0020;
+const uint8_t RS_PKT_SUBTYPE_NXS_MSG      = 0x0020;
 const uint8_t RS_PKT_SUBTYPE_SEARCH_REQ    = 0x0040;
-const uint8_t RS_PKT_SUBTYPE_SEARCH_RESP   = 0x0080;
+
+// possibility create second service to deal with this functionality
+
+const uint8_t RS_PKT_SUBTYPE_NXS_EXTENDED      = 0x0080; // in order to extend supported pkt subtypes
+const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_GRP   = 0x0001;
+const uint8_t RS_PKT_SUBTYPE_EXT_SEARCH_MSG   = 0x0002;
+const uint8_t RS_PKT_SUBTYPE_EXT_DELETE_GRP   = 0x0004;
+const uint8_t RS_PKT_SUBTYPE_EXT_DELETE_MSG   = 0x0008;
 
 
 /*!
@@ -75,6 +82,7 @@ class RsSyncGrp : public RsNxsItem {
 public:
 
     static const uint8_t FLAG_USE_SYNC_HASH;
+    static const uint8_t FLAG_ONLY_CURRENT; // only send most current sycn list
 
     RsSyncGrp(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_SYNC_GRP) { return;}
 
@@ -88,7 +96,7 @@ public:
 
 };
 
-typedef std::map<std::string, std::list<uint32_t> > SyncList;
+typedef std::map<std::string, std::list<RsTlvKeySignature> > SyncList;
 
 /*!
  * Use to send to peer list of grps
@@ -101,6 +109,7 @@ public:
 
     static const uint8_t FLAG_REQUEST;
     static const uint8_t FLAG_RESPONSE;
+    static const uint8_t FLAG_USE_SYNC_HASH;
 
     RsSyncGrpList(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_SYNC_GRP_LIST) { return ; }
 
@@ -111,7 +120,7 @@ public:
     uint8_t flag; // request or response
 
     /// groups held by sending peer
-     SyncList grps; // grpId/ n version pair
+     SyncList grps; // grpId/ sign pair
 
 };
 
@@ -120,18 +129,25 @@ public:
  * Each item corresponds to a group which needs to be
  * deserialised
  */
-class RsGrpResp : public RsNxsItem
+class RsNxsGrp : public RsNxsItem
 {
 
 public:
-    RsGrpResp(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_GRPS_RESP) { return; }
+
+    RsNxsGrp(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_GRP), grp(servtype) { return; }
 
 
     virtual void clear();
     virtual std::ostream &print(std::ostream &out, uint16_t indent);
 
-    std::list<RsTlvBinaryData*> grps; /// each entry corresponds to a group item
-
+    std::string grpId; /// group Id, needed to complete version Id (ncvi)
+    uint32_t timeStamp; /// UTC time, ncvi
+    RsTlvBinaryData grp; /// actual group data
+    RsTlvKeySignature adminSign; /// signature of admin (needed to complete version Id
+    RsTlvSecurityKeySet keys;
+    std::string identity;
+    RsTlvKeySignature idSign; /// identity sign if applicable
+    uint32_t grpFlag;
 };
 
 /*!
@@ -165,9 +181,9 @@ class RsSyncGrpMsgList : public RsNxsItem
 {
 public:
 
-    static const uint8_t FLAG_RESPONSE;
     static const uint8_t FLAG_REQUEST;
-
+    static const uint8_t FLAG_RESPONSE;
+    static const uint8_t FLAG_USE_SYNC_HASH;
     RsSyncGrpMsgList(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_SYNC_MSG_LIST) { return; }
 
 
@@ -185,17 +201,27 @@ public:
  * Used to respond to a RsGrpMsgsReq
  * with message items satisfying request
  */
-class RsGrpMsgResp : public RsNxsItem
+class RsNxsMsg : public RsNxsItem
 {
 public:
 
-    RsGrpMsgResp(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_MSG_RESP) { return; }
+
+
+    RsNxsMsg(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_MSG), msg(servtype) { return; }
 
 
     virtual void clear();
     virtual std::ostream &print(std::ostream &out, uint16_t indent);
 
-    std::list<RsTlvBinaryData*> msgs; // each entry corresponds to a message item
+    std::string grpId; /// group id, forms part of version id
+    std::string msgId; /// msg id
+    uint32_t msgFlag;
+    uint32_t timeStamp; /// UTC time create,
+    RsTlvBinaryData msg;
+    RsTlvKeySignature publishSign; /// publish signature
+    RsTlvKeySignature idSign; /// identity signature
+    std::string identity;
+
 };
 
 /*!
@@ -205,13 +231,35 @@ class RsNxsSearchReq : public RsNxsItem
 {
 public:
 
-    RsNxsSearchReq(uint16_t servtype): RsNxsItem(servtype, RS_PKT_SUBTYPE_SEARCH_REQ), searchTerm(servtype) { return; }
+    RsNxsSearchReq(uint16_t servtype): RsNxsItem(servtype, RS_PKT_SUBTYPE_SEARCH_REQ), serviceSearchItem(servtype) { return; }
+    virtual ~RsNxsSearchReq() { return;}
 
-    virtual void clear() {}
+    virtual void clear() { return;}
     virtual std::ostream &print(std::ostream &out, uint16_t indent) { return out; }
 
-    uint16_t token;
-    RsTlvBinaryData searchTerm; // service aware of item class
+    uint8_t nHops; /// how many peers to jump to
+    uint32_t token; // search token
+    RsTlvBinaryData serviceSearchItem; // service aware of item class
+    uint32_t expiration; // expiration date
+};
+
+
+/*!
+ * used to extend data types processed
+ */
+class RsNxsExtended : public RsNxsItem
+{
+
+public:
+
+    RsNxsExtended(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_NXS_EXTENDED), extData(servtype) { return; }
+    virtual ~RsNxsExtended() { return; }
+
+    virtual void clear() {}
+    virtual std::ostream &print(std::ostream &out, uint16_t indent);
+
+    RsTlvBinaryData extData;
+    uint32_t type;
 };
 
 
@@ -219,21 +267,71 @@ public:
  * Used to respond to a RsGrpSearchReq
  * with grpId/MsgIds that satisfy search request
  */
-class RsNxsSearchResp : public RsNxsItem
+class RsNxsSearchResultMsg
 {
 public:
 
-    RsNxsSearchResp(uint16_t servtype) : RsNxsItem(servtype, RS_PKT_SUBTYPE_SEARCH_RESP) { return; }
+    RsNxsSearchResultMsg() : context(0) { return;}
+    void clear() {}
+    std::ostream &print(std::ostream &out, uint16_t indent) { return out; }
 
+    uint32_t token; // search token to be redeemed
+    RsTlvBinaryData context; // used by client service
+    std::string msgId;
+    std::string grpId;
+    RsTlvKeySignature idSign;
 
-    virtual void clear() {}
-    virtual std::ostream &print(std::ostream &out, uint16_t indent) { return out; }
-
-    uint16_t token; // search token to be redeemed
-    std::list<RsTlvBinaryData> msgs; // msgs with search terms present
-    std::list<RsTlvBinaryData> grps; // grps with search terms present
+    uint32_t expiration; // expiration date
 };
 
+/*!
+ * Used to respond to a RsGrpSearchReq
+ * with grpId/MsgIds that satisfy search request
+ */
+class RsNxsSearchResultGrp
+{
+public:
+
+    RsNxsSearchResultGrp();
+    void clear() {}
+    std::ostream &print(std::ostream &out, uint16_t indent) { return out; }
+
+    uint32_t token; // search token to be redeemed
+
+    RsTlvBinaryData context; // used by client service
+
+    std::string grpId;
+    RsTlvKeySignature adminSign;
+
+    uint32_t expiration; // expiration date
+};
+
+
+class RsNxsDeleteMsg
+{
+public:
+
+    RsNxsDeleteMsg() { return; }
+
+    std::string msgId;
+    std::string grpId;
+    RsTlvKeySignature idSign;
+    RsTlvKeySignature deleteSign; // ( msgId + grpId + msg data ) sign //TODO: add warning not to place msgId+grpId in msg!
+
+};
+
+class RsNxsDeleteGrp
+{
+public:
+
+    RsNxsDeleteGrp() { return;}
+
+    std::string grpId;
+    RsTlvKeySignature idSign;
+    RsTlvKeySignature deleteSign; // (grpId + grp data) sign // TODO: add warning not to place grpId in msg
+};
+
+
 
 class RsNxsSerialiser : public RsSerialType
 {
@@ -263,11 +361,11 @@ private:
     virtual bool serialiseSyncGrpList(RsSyncGrpList *item, void *data, uint32_t *size);
     virtual RsSyncGrpList* deserialSyncGrpList(void *data, uint32_t *size);
 
-    /* for RS_PKT_SUBTYPE_GRPS_RESP */
+    /* for RS_PKT_SUBTYPE_NXS_GRP */
 
-    virtual uint32_t sizeGrpResp(RsGrpResp* item);
-    virtual bool serialiseGrpResp(RsGrpResp *item, void *data, uint32_t *size);
-    virtual RsGrpResp* deserialGrpResp(void *data, uint32_t *size);
+    virtual uint32_t sizeNxsGrp(RsNxsGrp* item);
+    virtual bool serialiseNxsGrp(RsNxsGrp *item, void *data, uint32_t *size);
+    virtual RsNxsGrp* deserialNxsGrp(void *data, uint32_t *size);
 
     /* for RS_PKT_SUBTYPE_SYNC_MSG */
 
@@ -281,28 +379,26 @@ private:
     virtual bool serialiseSynGrpMsgList(RsSyncGrpMsgList* item, void *data, uint32_t* size);
     virtual RsSyncGrpMsgList* deserialSyncGrpMsgList(void *data, uint32_t *size);
 
+    /* RS_PKT_SUBTYPE_NXS_MSG */
 
-    /* RS_PKT_SUBTYPE_MSG_RESP */
-
-    virtual uint32_t sizeGrpMsgResp(RsGrpMsgResp* item);
-    virtual bool serialiseGrpMsgResp(RsGrpMsgResp* item, void* data, uint32_t* size);
-    virtual RsGrpMsgResp* deserialGrpMsgResp(void *data, uint32_t *size);
+    virtual uint32_t sizeNxsMsg(RsNxsMsg* item);
+    virtual bool serialiseNxsMsg(RsNxsMsg* item, void* data, uint32_t* size);
+    virtual RsNxsMsg* deserialNxsMsg(void *data, uint32_t *size);
 
     /* RS_PKT_SUBTYPE_SEARCH_REQ */
-
     virtual uint32_t sizeNxsSearchReq(RsNxsSearchReq* item);
     virtual bool serialiseNxsSearchReq(RsNxsSearchReq* item, void* data, uint32_t* size);
-    virtual RsNxsSearchReq* deserialNxsSearchReq(void *data, uint32_t *size);
+    virtual RsNxsSearchReq* deserialNxsSearchReq(void* data, uint32_t *size);
 
-    /* RS_PKT_SUBTYPE_SEARCH_RESP */
-
-    virtual uint32_t sizeNxsSearchResp(RsNxsSearchResp *item);
-    virtual bool serialiseNxsSearchResp(RsNxsSearchResp *item, void *data, uint32_t *size);
-    virtual RsNxsSearchResp* deserialNxsSearchResp(void *data, uint32_t *size);
+    /* RS_PKT_SUBTYPE_EXTENDED */
+    virtual RsNxsExtended* deserialNxsExtended(void* data, uint32_t *size);
+    virtual uint32_t sizeNxsExtended(RsNxsExtended* item);
+    virtual bool serialiseNxsExtended(RsNxsExtended* item, void* data, uint32_t* size);
 
 private:
 
     const uint16_t SERVICE_TYPE;
 };
 
+
 #endif // RSNXSITEMS_H

libretroshare/src/util/retrodb.cc

@@ -29,7 +29,13 @@
 
 #include "retrodb.h"
 
-#define RETRODB_DEBUG
+//#define RETRODB_DEBUG
+
+void free_blob(void* dat){
+
+    char* c = (char*) dat;
+    delete[] c;
+}
 
 const uint8_t ContentValue::BOOL_TYPE  = 1;
 const uint8_t ContentValue::DATA_TYPE = 2;
@@ -217,6 +223,8 @@ bool RetroDb::sqlInsert(const std::string &table, const std::string& nullColumnH
     // build values part of insertion
     std::string qValues = "VALUES(";
     std::ostringstream oStrStream;
+    uint32_t index = 0;
+    std::list<RetroDbBlob> blobL;
 
     for(mit=keyTypeMap.begin(); mit!=keyTypeMap.end(); mit++){
 
@@ -246,8 +254,12 @@ bool RetroDb::sqlInsert(const std::string &table, const std::string& nullColumnH
                 char* value;
                 uint32_t len;
                 cv.getAsData(key, len, value);
-                oStrStream.write(value, len);
-                qValues += "'" + oStrStream.str() + "'";
+                RetroDbBlob b;
+                b.data = value;
+                b.length = len;
+                b.index = ++index;
+                blobL.push_back(b);
+                qValues += "?"; // parameter
                 break;
             }
         case ContentValue::STRING_TYPE:
@@ -297,10 +309,81 @@ bool RetroDb::sqlInsert(const std::string &table, const std::string& nullColumnH
 #endif
 
     // execute query
-    execSQL(sqlQuery);
+    execSQL_bind_blobs(sqlQuery, blobL);
     return true;
 }
 
+bool RetroDb::execSQL_bind_blobs(const std::string &query, std::list<RetroDbBlob> &blobs){
+
+    // prepare statement
+    sqlite3_stmt* stm = NULL;
+
+#ifdef RETRODB_DEBUG
+    std::cerr << "Query: " << query << std::endl;
+#endif
+
+    int rc = sqlite3_prepare_v2(mDb, query.c_str(), query.length(), &stm, NULL);
+
+    // check if there are any errors
+    if(rc != SQLITE_OK){
+        std::cerr << "RetroDb::execSQL(): Error preparing statement\n";
+        std::cerr << "Error code: " <<  sqlite3_errmsg(mDb)
+                  << std::endl;
+        return false;
+    }
+
+    std::list<RetroDbBlob>::iterator lit = blobs.begin();
+
+    for(; lit != blobs.end(); lit++){
+        const RetroDbBlob& b = *lit;
+        sqlite3_bind_blob(stm, b.index, b.data, b.length, free_blob);
+    }
+
+    uint32_t delta = 3;
+    time_t stamp = time(NULL), now = 0;
+    bool timeOut = false, ok = false;
+
+    while(!timeOut){
+
+        rc = sqlite3_step(stm);
+
+        if(rc == SQLITE_DONE){
+            ok = true;
+            break;
+        }
+
+        if(rc != SQLITE_BUSY){
+            ok = false;
+            break;
+        }
+
+        now = time(NULL);
+        delta = stamp - now;
+
+        if(delta > TIME_LIMIT){
+            ok = false;
+            timeOut = true;
+        }
+        // TODO add sleep so not to waste
+        // precious cycles
+    }
+
+    if(!ok){
+
+        if(rc == SQLITE_BUSY){
+            std::cerr << "RetroDb::execSQL()\n" ;
+            std::cerr << "SQL timed out!" << std::endl;
+        }else{
+            std::cerr << "RetroDb::execSQL(): Error executing statement (code: " << rc << ")\n";
+            std::cerr << "Sqlite Error msg: " <<  sqlite3_errmsg(mDb)
+                      << std::endl;
+        }
+    }
+
+    // finalise statement or else db cannot be closed
+    sqlite3_finalize(stm);
+    return ok;
+}
 
 bool RetroDb::sqlDelete(const std::string &tableName, const std::string &whereClause, const std::string &whereArgs){
 
@@ -641,7 +724,6 @@ ContentValue::ContentValue(){
 
 ContentValue::~ContentValue(){
 
-
 }
 
 ContentValue::ContentValue(ContentValue &from){

libretroshare/src/util/retrodb.h

@@ -149,6 +149,18 @@ public:
 
 private:
 
+    class RetroDbBlob{
+
+    public:
+
+        char* data;
+        uint32_t length;
+        uint32_t index;
+    };
+
+    bool execSQL_bind_blobs(const std::string &query, std::list<RetroDbBlob>& blobs);
+
+private:
 
     sqlite3* mDb;
 
@@ -260,6 +272,8 @@ public:
 
     /*!
      * Returns the value of the requested column as a String.
+     * data returned must be copied, as it is freed after RetroDb
+     * is closed or destroyed
      * @param columnIndex the zero-based index of the target column.
      * @return  the value of the column as pointer to raw data
      */

libretroshare/src/util/rsversion.h

@@ -1,17 +1,18 @@
-/*
- * rsversion.h
- *
- *  Created on: Jun 23, 2009
- *      Author: alexandrut
- */
-
-#include <string>
-
-#define LIB_VERSION "0.5.2a"
-#define SVN_REVISION "Revision 4550"
-
-namespace RsUtil {
-
-	std::string retroshareVersion();
-
-}
+/*
+ * rsversion.h
+ *
+ *  Created on: Jun 23, 2009
+ *      Author: alexandrut
+ */
+
+#include <string>
+
+#define LIB_VERSION "0.5.2a"
+#define SVN_REVISION "Revision: 4974"
+
+
+namespace RsUtil {
+
+	std::string retroshareVersion();
+
+}