merged with latest trunk

This commit is contained in:
csoler 2016-01-01 15:15:19 -05:00
commit 6ecd2991e7
2342 changed files with 29868 additions and 19573 deletions

View file

@ -27,6 +27,7 @@
#include "gxssecurity.h"
#include "pqi/authgpg.h"
#include "util/rsdir.h"
#include "util/rsmemory.h"
//#include "retroshare/rspeers.h"
/****
@ -749,7 +750,7 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in,
in_offset += size_net_ekl;
// Conservative limits to detect weird errors due to corrupted encoding.
if(eklen < 0 || eklen > 512 || eklen+in_offset > inlen)
if(eklen < 0 || eklen > 512 || eklen+in_offset > (int)inlen)
{
std::cerr << "Error while deserialising encryption key length: eklen = " << std::dec << eklen << ". Giving up decryption." << std::endl;
free(ek);
@ -809,104 +810,114 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in,
bool GxsSecurity::validateNxsGrp(const RsNxsGrp& grp, const RsTlvKeySignature& sign, const RsTlvSecurityKey& key)
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << std::endl;
std::cerr << "RsNxsGrp :";
std::cerr << std::endl;
grp.print(std::cerr, 10);
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << std::endl;
std::cerr << "RsNxsGrp :";
std::cerr << std::endl;
grp.print(std::cerr, 10);
std::cerr << std::endl;
#endif
RsGxsGrpMetaData& grpMeta = *(grp.metaData);
RsGxsGrpMetaData& grpMeta = *(grp.metaData);
/********************* check signature *******************/
/********************* check signature *******************/
/* check signature timeperiod */
if ((grpMeta.mPublishTs < key.startTS) || (key.endTS != 0 && grpMeta.mPublishTs > key.endTS))
{
/* check signature timeperiod */
if ((grpMeta.mPublishTs < key.startTS) || (key.endTS != 0 && grpMeta.mPublishTs > key.endTS))
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << " GxsSecurity::validateNxsMsg() TS out of range";
std::cerr << std::endl;
std::cerr << " GxsSecurity::validateNxsMsg() TS out of range";
std::cerr << std::endl;
#endif
return false;
}
return false;
}
/* decode key */
const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data;
long keylen = key.keyData.bin_len;
unsigned int siglen = sign.signData.bin_len;
unsigned char *sigbuf = (unsigned char *) sign.signData.bin_data;
/* decode key */
const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data;
long keylen = key.keyData.bin_len;
unsigned int siglen = sign.signData.bin_len;
unsigned char *sigbuf = (unsigned char *) sign.signData.bin_data;
#ifdef DISTRIB_DEBUG
std::cerr << "GxsSecurity::validateNxsMsg() Decode Key";
std::cerr << " keylen: " << keylen << " siglen: " << siglen;
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsMsg() Decode Key";
std::cerr << " keylen: " << keylen << " siglen: " << siglen;
std::cerr << std::endl;
#endif
/* extract admin key */
RSA *rsakey = (key.keyFlags & RSTLV_KEY_TYPE_FULL)? d2i_RSAPrivateKey(NULL, &(keyptr), keylen): d2i_RSAPublicKey(NULL, &(keyptr), keylen);
/* extract admin key */
RSA *rsakey = (key.keyFlags & RSTLV_KEY_TYPE_FULL)? d2i_RSAPrivateKey(NULL, &(keyptr), keylen): d2i_RSAPublicKey(NULL, &(keyptr), keylen);
if (!rsakey)
{
if (!rsakey)
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << " Invalid RSA Key";
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << " Invalid RSA Key";
std::cerr << std::endl;
key.print(std::cerr, 10);
key.print(std::cerr, 10);
#endif
}
}
std::vector<uint32_t> api_versions_to_check ;
api_versions_to_check.push_back(RS_GXS_GRP_META_DATA_VERSION_ID_0002) ; // put newest first, for debug info purpose
api_versions_to_check.push_back(RS_GXS_GRP_META_DATA_VERSION_ID_0001) ;
RsTlvKeySignatureSet signSet = grpMeta.signSet;
grpMeta.signSet.TlvClear();
int signOk =0;
EVP_PKEY *signKey = EVP_PKEY_new();
EVP_PKEY_assign_RSA(signKey, rsakey);
RsTlvKeySignatureSet signSet = grpMeta.signSet;
grpMeta.signSet.TlvClear();
for(uint32_t i=0;i<api_versions_to_check.size() && 0==signOk;++i)
{
uint32_t metaDataLen = grpMeta.serial_size(api_versions_to_check[i]);
uint32_t allGrpDataLen = metaDataLen + grp.grp.bin_len;
uint32_t metaDataLen = grpMeta.serial_size();
uint32_t allGrpDataLen = metaDataLen + grp.grp.bin_len;
char* metaData = new char[metaDataLen];
char* allGrpData = new char[allGrpDataLen]; // msgData + metaData
RsTemporaryMemory metaData(metaDataLen) ;
RsTemporaryMemory allGrpData(allGrpDataLen) ;// msgData + metaData
grpMeta.serialise(metaData, metaDataLen);
grpMeta.serialise(metaData, metaDataLen,api_versions_to_check[i]);
// copy msg data and meta in allmsgData buffer
memcpy(allGrpData, grp.grp.bin_data, grp.grp.bin_len);
memcpy(allGrpData+(grp.grp.bin_len), metaData, metaDataLen);
// copy msg data and meta in allmsgData buffer
memcpy(allGrpData, grp.grp.bin_data, grp.grp.bin_len);
memcpy(allGrpData+(grp.grp.bin_len), metaData, metaDataLen);
delete[] metaData ;
/* calc and check signature */
EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
EVP_PKEY *signKey = EVP_PKEY_new();
EVP_PKEY_assign_RSA(signKey, rsakey);
EVP_VerifyInit(mdctx, EVP_sha1());
EVP_VerifyUpdate(mdctx, allGrpData, allGrpDataLen);
signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey);
EVP_MD_CTX_destroy(mdctx);
/* calc and check signature */
EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
if(i>0)
std::cerr << "(WW) Checking group signature with old api version " << i+1 << " : tag " << std::hex << api_versions_to_check[i] << std::dec << " result: " << signOk << std::endl;
}
EVP_VerifyInit(mdctx, EVP_sha1());
EVP_VerifyUpdate(mdctx, allGrpData, allGrpDataLen);
int signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey);
/* clean up */
EVP_PKEY_free(signKey);
delete[] allGrpData ;
// restore data
/* clean up */
EVP_PKEY_free(signKey);
EVP_MD_CTX_destroy(mdctx);
grpMeta.signSet = signSet;
grpMeta.signSet = signSet;
if (signOk == 1)
{
if (signOk == 1)
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp() Signature OK";
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsGrp() Signature OK";
std::cerr << std::endl;
#endif
return true;
}
return true;
}
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp() Signature invalid";
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsGrp() Signature invalid";
std::cerr << std::endl;
#endif
return false;
return false;
}

View file

@ -49,19 +49,19 @@
#define MSG_INDEX_GRPID std::string("INDEX_MESSAGES_GRPID")
// generic
#define KEY_NXS_DATA std::string("nxsData")
#define KEY_NXS_DATA_LEN std::string("nxsDataLen")
#define KEY_NXS_IDENTITY std::string("identity")
#define KEY_GRP_ID std::string("grpId")
#define KEY_ORIG_GRP_ID std::string("origGrpId")
#define KEY_PARENT_GRP_ID std::string("parentGrpId")
#define KEY_SIGN_SET std::string("signSet")
#define KEY_TIME_STAMP std::string("timeStamp")
#define KEY_NXS_FLAGS std::string("flags")
#define KEY_NXS_META std::string("meta")
#define KEY_NXS_DATA std::string("nxsData")
#define KEY_NXS_DATA_LEN std::string("nxsDataLen")
#define KEY_NXS_IDENTITY std::string("identity")
#define KEY_GRP_ID std::string("grpId")
#define KEY_ORIG_GRP_ID std::string("origGrpId")
#define KEY_PARENT_GRP_ID std::string("parentGrpId")
#define KEY_SIGN_SET std::string("signSet")
#define KEY_TIME_STAMP std::string("timeStamp")
#define KEY_NXS_FLAGS std::string("flags")
#define KEY_NXS_META std::string("meta")
#define KEY_NXS_SERV_STRING std::string("serv_str")
#define KEY_NXS_HASH std::string("hash")
#define KEY_RECV_TS std::string("recv_time_stamp")
#define KEY_NXS_HASH std::string("hash")
#define KEY_RECV_TS std::string("recv_time_stamp")
// remove later
#define KEY_NXS_FILE_OLD std::string("nxsFile")
@ -94,8 +94,8 @@
#define KEY_MSG_NAME std::string("msgName")
// msg local
#define KEY_MSG_STATUS std::string("msgStatus")
#define KEY_CHILD_TS std::string("childTs")
#define KEY_MSG_STATUS std::string("msgStatus")
#define KEY_CHILD_TS std::string("childTs")
// database release columns
#define KEY_DATABASE_RELEASE_ID std::string("id")
@ -129,22 +129,22 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d
initialise(isNewDatabase);
// for retrieving msg meta
mColMsgMeta_GrpId = addColumn(mMsgMetaColumns, KEY_GRP_ID);
mColMsgMeta_TimeStamp = addColumn(mMsgMetaColumns, KEY_TIME_STAMP);
mColMsgMeta_NxsFlags = addColumn(mMsgMetaColumns, KEY_NXS_FLAGS);
mColMsgMeta_SignSet = addColumn(mMsgMetaColumns, KEY_SIGN_SET);
mColMsgMeta_NxsIdentity = addColumn(mMsgMetaColumns, KEY_NXS_IDENTITY);
mColMsgMeta_NxsHash = addColumn(mMsgMetaColumns, KEY_NXS_HASH);
mColMsgMeta_MsgId = addColumn(mMsgMetaColumns, KEY_MSG_ID);
mColMsgMeta_OrigMsgId = addColumn(mMsgMetaColumns, KEY_ORIG_MSG_ID);
mColMsgMeta_MsgStatus = addColumn(mMsgMetaColumns, KEY_MSG_STATUS);
mColMsgMeta_ChildTs = addColumn(mMsgMetaColumns, KEY_CHILD_TS);
mColMsgMeta_MsgParentId = addColumn(mMsgMetaColumns, KEY_MSG_PARENT_ID);
mColMsgMeta_MsgThreadId = addColumn(mMsgMetaColumns, KEY_MSG_THREAD_ID);
mColMsgMeta_Name = addColumn(mMsgMetaColumns, KEY_MSG_NAME);
mColMsgMeta_GrpId = addColumn(mMsgMetaColumns, KEY_GRP_ID);
mColMsgMeta_TimeStamp = addColumn(mMsgMetaColumns, KEY_TIME_STAMP);
mColMsgMeta_NxsFlags = addColumn(mMsgMetaColumns, KEY_NXS_FLAGS);
mColMsgMeta_SignSet = addColumn(mMsgMetaColumns, KEY_SIGN_SET);
mColMsgMeta_NxsIdentity = addColumn(mMsgMetaColumns, KEY_NXS_IDENTITY);
mColMsgMeta_NxsHash = addColumn(mMsgMetaColumns, KEY_NXS_HASH);
mColMsgMeta_MsgId = addColumn(mMsgMetaColumns, KEY_MSG_ID);
mColMsgMeta_OrigMsgId = addColumn(mMsgMetaColumns, KEY_ORIG_MSG_ID);
mColMsgMeta_MsgStatus = addColumn(mMsgMetaColumns, KEY_MSG_STATUS);
mColMsgMeta_ChildTs = addColumn(mMsgMetaColumns, KEY_CHILD_TS);
mColMsgMeta_MsgParentId = addColumn(mMsgMetaColumns, KEY_MSG_PARENT_ID);
mColMsgMeta_MsgThreadId = addColumn(mMsgMetaColumns, KEY_MSG_THREAD_ID);
mColMsgMeta_Name = addColumn(mMsgMetaColumns, KEY_MSG_NAME);
mColMsgMeta_NxsServString = addColumn(mMsgMetaColumns, KEY_NXS_SERV_STRING);
mColMsgMeta_RecvTs = addColumn(mMsgMetaColumns, KEY_RECV_TS);
mColMsgMeta_NxsDataLen = addColumn(mMsgMetaColumns, KEY_NXS_DATA_LEN);
mColMsgMeta_RecvTs = addColumn(mMsgMetaColumns, KEY_RECV_TS);
mColMsgMeta_NxsDataLen = addColumn(mMsgMetaColumns, KEY_NXS_DATA_LEN);
// for retrieving actual data
mColMsg_GrpId = addColumn(mMsgColumns, KEY_GRP_ID);
@ -152,38 +152,48 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d
mColMsg_MetaData = addColumn(mMsgColumns, KEY_NXS_META);
mColMsg_MsgId = addColumn(mMsgColumns, KEY_MSG_ID);
// for retrieving msg data with meta
mMsgColumnsWithMeta = mMsgColumns;
mColMsg_WithMetaOffset = mMsgColumnsWithMeta.size();
mMsgColumnsWithMeta.insert(mMsgColumnsWithMeta.end(), mMsgMetaColumns.begin(), mMsgMetaColumns.end());
// for retrieving grp meta data
mColGrpMeta_GrpId = addColumn(mGrpMetaColumns, KEY_GRP_ID);
mColGrpMeta_TimeStamp = addColumn(mGrpMetaColumns, KEY_TIME_STAMP);
mColGrpMeta_NxsFlags = addColumn(mGrpMetaColumns, KEY_NXS_FLAGS);
mColGrpMeta_GrpId = addColumn(mGrpMetaColumns, KEY_GRP_ID);
mColGrpMeta_TimeStamp = addColumn(mGrpMetaColumns, KEY_TIME_STAMP);
mColGrpMeta_NxsFlags = addColumn(mGrpMetaColumns, KEY_NXS_FLAGS);
// mColGrpMeta_SignSet = addColumn(mGrpMetaColumns, KEY_SIGN_SET);
mColGrpMeta_NxsIdentity = addColumn(mGrpMetaColumns, KEY_NXS_IDENTITY);
mColGrpMeta_NxsHash = addColumn(mGrpMetaColumns, KEY_NXS_HASH);
mColGrpMeta_KeySet = addColumn(mGrpMetaColumns, KEY_KEY_SET);
mColGrpMeta_SubscrFlag = addColumn(mGrpMetaColumns, KEY_GRP_SUBCR_FLAG);
mColGrpMeta_Pop = addColumn(mGrpMetaColumns, KEY_GRP_POP);
mColGrpMeta_MsgCount = addColumn(mGrpMetaColumns, KEY_MSG_COUNT);
mColGrpMeta_Status = addColumn(mGrpMetaColumns, KEY_GRP_STATUS);
mColGrpMeta_Name = addColumn(mGrpMetaColumns, KEY_GRP_NAME);
mColGrpMeta_LastPost = addColumn(mGrpMetaColumns, KEY_GRP_LAST_POST);
mColGrpMeta_OrigGrpId = addColumn(mGrpMetaColumns, KEY_ORIG_GRP_ID);
mColGrpMeta_ServString = addColumn(mGrpMetaColumns, KEY_NXS_SERV_STRING);
mColGrpMeta_SignFlags = addColumn(mGrpMetaColumns, KEY_GRP_SIGN_FLAGS);
mColGrpMeta_CircleId = addColumn(mGrpMetaColumns, KEY_GRP_CIRCLE_ID);
mColGrpMeta_CircleType = addColumn(mGrpMetaColumns, KEY_GRP_CIRCLE_TYPE);
mColGrpMeta_NxsHash = addColumn(mGrpMetaColumns, KEY_NXS_HASH);
mColGrpMeta_KeySet = addColumn(mGrpMetaColumns, KEY_KEY_SET);
mColGrpMeta_SubscrFlag = addColumn(mGrpMetaColumns, KEY_GRP_SUBCR_FLAG);
mColGrpMeta_Pop = addColumn(mGrpMetaColumns, KEY_GRP_POP);
mColGrpMeta_MsgCount = addColumn(mGrpMetaColumns, KEY_MSG_COUNT);
mColGrpMeta_Status = addColumn(mGrpMetaColumns, KEY_GRP_STATUS);
mColGrpMeta_Name = addColumn(mGrpMetaColumns, KEY_GRP_NAME);
mColGrpMeta_LastPost = addColumn(mGrpMetaColumns, KEY_GRP_LAST_POST);
mColGrpMeta_OrigGrpId = addColumn(mGrpMetaColumns, KEY_ORIG_GRP_ID);
mColGrpMeta_ServString = addColumn(mGrpMetaColumns, KEY_NXS_SERV_STRING);
mColGrpMeta_SignFlags = addColumn(mGrpMetaColumns, KEY_GRP_SIGN_FLAGS);
mColGrpMeta_CircleId = addColumn(mGrpMetaColumns, KEY_GRP_CIRCLE_ID);
mColGrpMeta_CircleType = addColumn(mGrpMetaColumns, KEY_GRP_CIRCLE_TYPE);
mColGrpMeta_InternCircle = addColumn(mGrpMetaColumns, KEY_GRP_INTERNAL_CIRCLE);
mColGrpMeta_Originator = addColumn(mGrpMetaColumns, KEY_GRP_ORIGINATOR);
mColGrpMeta_Originator = addColumn(mGrpMetaColumns, KEY_GRP_ORIGINATOR);
mColGrpMeta_AuthenFlags = addColumn(mGrpMetaColumns, KEY_GRP_AUTHEN_FLAGS);
mColGrpMeta_ParentGrpId = addColumn(mGrpMetaColumns, KEY_PARENT_GRP_ID);
mColGrpMeta_RecvTs = addColumn(mGrpMetaColumns, KEY_RECV_TS);
mColGrpMeta_RepCutoff = addColumn(mGrpMetaColumns, KEY_GRP_REP_CUTOFF);
mColGrpMeta_NxsDataLen = addColumn(mGrpMetaColumns, KEY_NXS_DATA_LEN);
mColGrpMeta_RecvTs = addColumn(mGrpMetaColumns, KEY_RECV_TS);
mColGrpMeta_RepCutoff = addColumn(mGrpMetaColumns, KEY_GRP_REP_CUTOFF);
mColGrpMeta_NxsDataLen = addColumn(mGrpMetaColumns, KEY_NXS_DATA_LEN);
// for retrieving actual grp data
mColGrp_GrpId = addColumn(mGrpColumns, KEY_GRP_ID);
mColGrp_NxsData = addColumn(mGrpColumns, KEY_NXS_DATA);
mColGrp_MetaData = addColumn(mGrpColumns, KEY_NXS_META);
// for retrieving grp data with meta
mGrpColumnsWithMeta = mGrpColumns;
mColGrp_WithMetaOffset = mGrpColumnsWithMeta.size();
mGrpColumnsWithMeta.insert(mGrpColumnsWithMeta.end(), mGrpMetaColumns.begin(), mGrpMetaColumns.end());
// Group id columns
mColGrpId_GrpId = addColumn(mGrpIdColumn, KEY_GRP_ID);
@ -474,7 +484,7 @@ bool RsDataService::finishReleaseUpdate(int release, bool result)
return result;
}
RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c)
RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c, int colOffset)
{
#ifdef RS_DATA_SERVICE_DEBUG
std::cerr << "RsDataService::locked_getGrpMeta()";
@ -492,27 +502,27 @@ RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c)
// grpId
std::string tempId;
c.getString(mColGrpMeta_GrpId, tempId);
c.getString(mColGrpMeta_GrpId + colOffset, tempId);
grpMeta->mGroupId = RsGxsGroupId(tempId);
c.getString(mColGrpMeta_NxsIdentity, tempId);
c.getString(mColGrpMeta_NxsIdentity + colOffset, tempId);
grpMeta->mAuthorId = RsGxsId(tempId);
c.getString(mColGrpMeta_Name, grpMeta->mGroupName);
c.getString(mColGrpMeta_OrigGrpId, tempId);
c.getString(mColGrpMeta_Name + colOffset, grpMeta->mGroupName);
c.getString(mColGrpMeta_OrigGrpId + colOffset, tempId);
grpMeta->mOrigGrpId = RsGxsGroupId(tempId);
c.getString(mColGrpMeta_ServString, grpMeta->mServiceString);
c.getString(mColGrpMeta_ServString + colOffset, grpMeta->mServiceString);
std::string temp;
c.getString(mColGrpMeta_NxsHash, temp);
c.getString(mColGrpMeta_NxsHash + colOffset, temp);
grpMeta->mHash = RsFileHash(temp);
grpMeta->mReputationCutOff = c.getInt32(mColGrpMeta_RepCutoff);
grpMeta->mSignFlags = c.getInt32(mColGrpMeta_SignFlags);
grpMeta->mReputationCutOff = c.getInt32(mColGrpMeta_RepCutoff + colOffset);
grpMeta->mSignFlags = c.getInt32(mColGrpMeta_SignFlags + colOffset);
grpMeta->mPublishTs = c.getInt32(mColGrpMeta_TimeStamp);
grpMeta->mGroupFlags = c.getInt32(mColGrpMeta_NxsFlags);
grpMeta->mGrpSize = c.getInt32(mColGrpMeta_NxsDataLen);
grpMeta->mPublishTs = c.getInt32(mColGrpMeta_TimeStamp + colOffset);
grpMeta->mGroupFlags = c.getInt32(mColGrpMeta_NxsFlags + colOffset);
grpMeta->mGrpSize = c.getInt32(mColGrpMeta_NxsDataLen + colOffset);
offset = 0; data = NULL; data_len = 0;
data = (char*)c.getData(mColGrpMeta_KeySet, data_len);
data = (char*)c.getData(mColGrpMeta_KeySet + colOffset, data_len);
if(data)
ok &= grpMeta->keys.GetTlv(data, data_len, &offset);
@ -520,22 +530,22 @@ RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c)
grpMeta->keys.TlvClear() ;
// local meta
grpMeta->mSubscribeFlags = c.getInt32(mColGrpMeta_SubscrFlag);
grpMeta->mPop = c.getInt32(mColGrpMeta_Pop);
grpMeta->mVisibleMsgCount = c.getInt32(mColGrpMeta_MsgCount);
grpMeta->mLastPost = c.getInt32(mColGrpMeta_LastPost);
grpMeta->mGroupStatus = c.getInt32(mColGrpMeta_Status);
grpMeta->mSubscribeFlags = c.getInt32(mColGrpMeta_SubscrFlag + colOffset);
grpMeta->mPop = c.getInt32(mColGrpMeta_Pop + colOffset);
grpMeta->mVisibleMsgCount = c.getInt32(mColGrpMeta_MsgCount + colOffset);
grpMeta->mLastPost = c.getInt32(mColGrpMeta_LastPost + colOffset);
grpMeta->mGroupStatus = c.getInt32(mColGrpMeta_Status + colOffset);
c.getString(mColGrpMeta_CircleId, tempId);
c.getString(mColGrpMeta_CircleId + colOffset, tempId);
grpMeta->mCircleId = RsGxsCircleId(tempId);
grpMeta->mCircleType = c.getInt32(mColGrpMeta_CircleType);
c.getString(mColGrpMeta_InternCircle, tempId);
grpMeta->mCircleType = c.getInt32(mColGrpMeta_CircleType + colOffset);
c.getString(mColGrpMeta_InternCircle + colOffset, tempId);
grpMeta->mInternalCircle = RsGxsCircleId(tempId);
std::string s ; c.getString(mColGrpMeta_Originator, s) ;
std::string s ; c.getString(mColGrpMeta_Originator + colOffset, s) ;
grpMeta->mOriginator = RsPeerId(s);
grpMeta->mAuthenFlags = c.getInt32(mColGrpMeta_AuthenFlags);
grpMeta->mRecvTS = c.getInt32(mColGrpMeta_RecvTs);
grpMeta->mAuthenFlags = c.getInt32(mColGrpMeta_AuthenFlags + colOffset);
grpMeta->mRecvTS = c.getInt32(mColGrpMeta_RecvTs + colOffset);
c.getString(mColGrpMeta_ParentGrpId, tempId);
@ -591,7 +601,7 @@ RsNxsGrp* RsDataService::locked_getGroup(RetroCursor &c)
return NULL;
}
RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c)
RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c, int colOffset)
{
RsGxsMsgMetaData* msgMeta = new RsGxsMsgMetaData();
@ -602,44 +612,43 @@ RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c)
char* data = NULL;
std::string gId;
c.getString(mColMsgMeta_GrpId, gId);
c.getString(mColMsgMeta_GrpId + colOffset, gId);
msgMeta->mGroupId = RsGxsGroupId(gId);
std::string temp;
c.getString(mColMsgMeta_MsgId, temp);
c.getString(mColMsgMeta_MsgId + colOffset, temp);
msgMeta->mMsgId = RsGxsMessageId(temp);
// without these, a msg is meaningless
ok &= (!msgMeta->mGroupId.isNull()) && (!msgMeta->mMsgId.isNull());
c.getString(mColMsgMeta_OrigMsgId, temp);
c.getString(mColMsgMeta_OrigMsgId + colOffset, temp);
msgMeta->mOrigMsgId = RsGxsMessageId(temp);
c.getString(mColMsgMeta_NxsIdentity, temp);
c.getString(mColMsgMeta_NxsIdentity + colOffset, temp);
msgMeta->mAuthorId = RsGxsId(temp);
c.getString(mColMsgMeta_Name, msgMeta->mMsgName);
c.getString(mColMsgMeta_NxsServString, msgMeta->mServiceString);
c.getString(mColMsgMeta_Name + colOffset, msgMeta->mMsgName);
c.getString(mColMsgMeta_NxsServString + colOffset, msgMeta->mServiceString);
c.getString(mColMsgMeta_NxsHash, temp);
c.getString(mColMsgMeta_NxsHash + colOffset, temp);
msgMeta->mHash = RsFileHash(temp);
msgMeta->recvTS = c.getInt32(mColMsgMeta_RecvTs);
msgMeta->recvTS = c.getInt32(mColMsgMeta_RecvTs + colOffset);
offset = 0;
data = (char*)c.getData(mColMsgMeta_SignSet, data_len);
data = (char*)c.getData(mColMsgMeta_SignSet + colOffset, data_len);
msgMeta->signSet.GetTlv(data, data_len, &offset);
msgMeta->mMsgSize = c.getInt32(mColMsgMeta_NxsDataLen);
msgMeta->mMsgSize = c.getInt32(mColMsgMeta_NxsDataLen + colOffset);
msgMeta->mMsgFlags = c.getInt32(mColMsgMeta_NxsFlags);
msgMeta->mPublishTs = c.getInt32(mColMsgMeta_TimeStamp);
msgMeta->mMsgFlags = c.getInt32(mColMsgMeta_NxsFlags + colOffset);
msgMeta->mPublishTs = c.getInt32(mColMsgMeta_TimeStamp + colOffset);
offset = 0; data_len = 0;
// thread and parent id
c.getString(mColMsgMeta_MsgThreadId, temp);
c.getString(mColMsgMeta_MsgThreadId + colOffset, temp);
msgMeta->mThreadId = RsGxsMessageId(temp);
c.getString(mColMsgMeta_MsgParentId, temp);
c.getString(mColMsgMeta_MsgParentId + colOffset, temp);
msgMeta->mParentId = RsGxsMessageId(temp);
// local meta
msgMeta->mMsgStatus = c.getInt32(mColMsgMeta_MsgStatus);
msgMeta->mChildTs = c.getInt32(mColMsgMeta_ChildTs);
msgMeta->mMsgStatus = c.getInt32(mColMsgMeta_MsgStatus + colOffset);
msgMeta->mChildTs = c.getInt32(mColMsgMeta_ChildTs + colOffset);
if(ok)
return msgMeta;
@ -1020,13 +1029,13 @@ int RsDataService::retrieveNxsGrps(std::map<RsGxsGroupId, RsNxsGrp *> &grp, bool
if(grp.empty()){
RsStackMutex stack(mDbMutex);
RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpColumns, "", "");
RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, withMeta ? mGrpColumnsWithMeta : mGrpColumns, "", "");
if(c)
{
std::vector<RsNxsGrp*> grps;
locked_retrieveGroups(c, grps);
locked_retrieveGroups(c, grps, withMeta ? mColGrp_WithMetaOffset : 0);
std::vector<RsNxsGrp*>::iterator vit = grps.begin();
#ifdef RS_DATA_SERVICE_DEBUG_TIME
@ -1051,12 +1060,12 @@ int RsDataService::retrieveNxsGrps(std::map<RsGxsGroupId, RsNxsGrp *> &grp, bool
for(; mit != grp.end(); ++mit)
{
const RsGxsGroupId& grpId = mit->first;
RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpColumns, "grpId='" + grpId.toStdString() + "'", "");
RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, withMeta ? mGrpColumnsWithMeta : mGrpColumns, "grpId='" + grpId.toStdString() + "'", "");
if(c)
{
std::vector<RsNxsGrp*> grps;
locked_retrieveGroups(c, grps);
locked_retrieveGroups(c, grps, withMeta ? mColGrp_WithMetaOffset : 0);
if(!grps.empty())
{
@ -1085,38 +1094,10 @@ int RsDataService::retrieveNxsGrps(std::map<RsGxsGroupId, RsNxsGrp *> &grp, bool
std::cerr << "RsDataService::retrieveNxsGrps() " << mDbName << ", Requests: " << requestedGroups << ", Results: " << resultCount << ", Time: " << timer.duration() << std::endl;
#endif
if(withMeta && !grp.empty())
{
std::map<RsGxsGroupId, RsGxsGrpMetaData*> metaMap;
std::map<RsGxsGroupId, RsNxsGrp *>::iterator mit = grp.begin();
for(; mit != grp.end(); ++mit)
metaMap.insert(std::make_pair(mit->first, (RsGxsGrpMetaData*)(NULL)));
retrieveGxsGrpMetaData(metaMap);
mit = grp.begin();
for(; mit != grp.end(); ++mit)
{
RsNxsGrp* grpPtr = grp[mit->first];
grpPtr->metaData = metaMap[mit->first];
#ifdef RS_DATA_SERVICE_DEBUG
std::cerr << "RsDataService::retrieveNxsGrps() GrpId: " << mit->first.toStdString();
std::cerr << " CircleType: " << (uint32_t) grpPtr->metaData->mCircleType;
std::cerr << " CircleId: " << grpPtr->metaData->mCircleId.toStdString();
std::cerr << std::endl;
#endif
}
#ifdef RS_DATA_SERVICE_DEBUG_TIME
std::cerr << "RsDataService::retrieveNxsGrps() " << mDbName << ", Time with meta: " << timer.duration() << std::endl;
#endif
}
return 1;
}
void RsDataService::locked_retrieveGroups(RetroCursor* c, std::vector<RsNxsGrp*>& grps){
void RsDataService::locked_retrieveGroups(RetroCursor* c, std::vector<RsNxsGrp*>& grps, int metaOffset){
if(c){
bool valid = c->moveToFirst();
@ -1127,6 +1108,9 @@ void RsDataService::locked_retrieveGroups(RetroCursor* c, std::vector<RsNxsGrp*>
// only add the latest grp info
if(g)
{
if (metaOffset) {
g->metaData = locked_getGrpMeta(*c, metaOffset);
}
grps.push_back(g);
}
valid = c->moveToNext();
@ -1143,8 +1127,6 @@ int RsDataService::retrieveNxsMsgs(const GxsMsgReq &reqIds, GxsMsgResult &msg, b
GxsMsgReq::const_iterator mit = reqIds.begin();
GxsMsgReq metaReqIds;// collects metaReqIds if needed
for(; mit != reqIds.end(); ++mit)
{
@ -1158,15 +1140,11 @@ int RsDataService::retrieveNxsMsgs(const GxsMsgReq &reqIds, GxsMsgResult &msg, b
RsStackMutex stack(mDbMutex);
RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", "");
RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, withMeta ? mMsgColumnsWithMeta : mMsgColumns, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", "");
if(c)
{
locked_retrieveMessages(c, msgSet);
#ifdef RS_DATA_SERVICE_DEBUG_TIME
resultCount += msgSet.size();
#endif
locked_retrieveMessages(c, msgSet, withMeta ? mColMsg_WithMetaOffset : 0);
}
delete c;
@ -1180,36 +1158,24 @@ int RsDataService::retrieveNxsMsgs(const GxsMsgReq &reqIds, GxsMsgResult &msg, b
RsStackMutex stack(mDbMutex);
RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, mMsgColumns, KEY_GRP_ID+ "='" + grpId.toStdString()
RetroCursor* c = mDb->sqlQuery(MSG_TABLE_NAME, withMeta ? mMsgColumnsWithMeta : mMsgColumns, KEY_GRP_ID+ "='" + grpId.toStdString()
+ "' AND " + KEY_MSG_ID + "='" + msgId.toStdString() + "'", "");
if(c)
{
locked_retrieveMessages(c, msgSet);
#ifdef RS_DATA_SERVICE_DEBUG_TIME
resultCount += c->getResultCount();
#endif
locked_retrieveMessages(c, msgSet, withMeta ? mColMsg_WithMetaOffset : 0);
}
delete c;
}
}
#ifdef RS_DATA_SERVICE_DEBUG_TIME
resultCount += msgSet.size();
#endif
msg[grpId] = msgSet;
if(withMeta)
{
std::vector<RsGxsMessageId> msgIds;
std::vector<RsNxsMsg*>::iterator lit = msgSet.begin(),
lit_end = msgSet.end();
for(; lit != lit_end; ++lit)
msgIds.push_back( (*lit)->msgId );
metaReqIds[grpId] = msgIds;
}
msgSet.clear();
}
@ -1217,73 +1183,19 @@ int RsDataService::retrieveNxsMsgs(const GxsMsgReq &reqIds, GxsMsgResult &msg, b
std::cerr << "RsDataService::retrieveNxsMsgs() " << mDbName << ", Requests: " << reqIds.size() << ", Results: " << resultCount << ", Time: " << timer.duration() << std::endl;
#endif
// tres expensive !?
if(withMeta)
{
GxsMsgMetaResult metaResult;
// request with meta ids so there is no chance of
// a mem leak being left over
retrieveGxsMsgMetaData(metaReqIds, metaResult);
GxsMsgResult::iterator mit2 = msg.begin(), mit2_end = msg.end();
for(; mit2 != mit2_end; ++mit2)
{
const RsGxsGroupId& grpId = mit2->first;
std::vector<RsNxsMsg*>& msgV = msg[grpId];
std::vector<RsNxsMsg*>::iterator lit = msgV.begin(),
lit_end = msgV.end();
// as retrieval only attempts to retrieve what was found this elimiates chance
// of a memory fault as all are assigned
for(; lit != lit_end; ++lit)
{
std::vector<RsGxsMsgMetaData*>& msgMetaV = metaResult[grpId];
std::vector<RsGxsMsgMetaData*>::iterator meta_lit = msgMetaV.begin();
RsNxsMsg* msgPtr = *lit;
for(; meta_lit != msgMetaV.end(); )
{
RsGxsMsgMetaData* meta = *meta_lit;
if(meta->mMsgId == msgPtr->msgId)
{
msgPtr->metaData = meta;
meta_lit = msgMetaV.erase(meta_lit);
}else{
++meta_lit;
}
}
}
std::vector<RsGxsMsgMetaData*>& msgMetaV = metaResult[grpId];
std::vector<RsGxsMsgMetaData*>::iterator meta_lit;
// clean up just in case, should not go in here
for(meta_lit = msgMetaV.begin(); meta_lit !=
msgMetaV.end(); )
{
RsGxsMsgMetaData* meta = *meta_lit;
delete meta;
meta_lit = msgMetaV.erase(meta_lit);
}
}
#ifdef RS_DATA_SERVICE_DEBUG_TIME
std::cerr << "RsDataService::retrieveNxsMsgs() " << mDbName << ", Time with meta: " << timer.duration() << std::endl;
#endif
}
return 1;
}
void RsDataService::locked_retrieveMessages(RetroCursor *c, std::vector<RsNxsMsg *> &msgs)
void RsDataService::locked_retrieveMessages(RetroCursor *c, std::vector<RsNxsMsg *> &msgs, int metaOffset)
{
bool valid = c->moveToFirst();
while(valid){
RsNxsMsg* m = locked_getMessage(*c);
if(m){
if (metaOffset) {
m->metaData = locked_getMsgMeta(*c, metaOffset);
}
msgs.push_back(m);
}
@ -1318,10 +1230,6 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes
if (c)
{
locked_retrieveMsgMeta(c, metaSet);
#ifdef RS_DATA_SERVICE_DEBUG_TIME
resultCount += metaSet.size();
#endif
}
}else{
@ -1336,14 +1244,14 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes
if (c)
{
locked_retrieveMsgMeta(c, metaSet);
#ifdef RS_DATA_SERVICE_DEBUG_TIME
resultCount += c->getResultCount();
#endif
}
}
}
#ifdef RS_DATA_SERVICE_DEBUG_TIME
resultCount += metaSet.size();
#endif
msgMeta[grpId] = metaSet;
}
@ -1361,7 +1269,7 @@ void RsDataService::locked_retrieveMsgMeta(RetroCursor *c, std::vector<RsGxsMsgM
{
bool valid = c->moveToFirst();
while(valid){
RsGxsMsgMetaData* m = locked_getMsgMeta(*c);
RsGxsMsgMetaData* m = locked_getMsgMeta(*c, 0);
if(m != NULL)
msgMeta.push_back(m);
@ -1402,7 +1310,7 @@ int RsDataService::retrieveGxsGrpMetaData(std::map<RsGxsGroupId, RsGxsGrpMetaDat
while(valid)
{
RsGxsGrpMetaData* g = locked_getGrpMeta(*c);
RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0);
if(g)
{
grp[g->mGroupId] = g;
@ -1431,7 +1339,7 @@ int RsDataService::retrieveGxsGrpMetaData(std::map<RsGxsGroupId, RsGxsGrpMetaDat
while(valid)
{
RsGxsGrpMetaData* g = locked_getGrpMeta(*c);
RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0);
if(g)
{

View file

@ -181,14 +181,15 @@ private:
* @param c cursor to result set
* @param msgs messages retrieved from cursor are stored here
*/
void locked_retrieveMessages(RetroCursor* c, std::vector<RsNxsMsg*>& msgs);
void locked_retrieveMessages(RetroCursor* c, std::vector<RsNxsMsg*>& msgs, int metaOffset);
/*!
* Retrieves all the grp results from a cursor
* @param c cursor to result set
* @param grps groups retrieved from cursor are stored here
* @param withMeta this initialise the metaData member of the nxsgroups retrieved
*/
void locked_retrieveGroups(RetroCursor* c, std::vector<RsNxsGrp*>& grps);
void locked_retrieveGroups(RetroCursor* c, std::vector<RsNxsGrp*>& grps, int metaOffset);
/*!
* Retrieves all the msg meta results from a cursor
@ -201,13 +202,13 @@ private:
* extracts a msg meta item from a cursor at its
* current position
*/
RsGxsMsgMetaData* locked_getMsgMeta(RetroCursor& c);
RsGxsMsgMetaData* locked_getMsgMeta(RetroCursor& c, int colOffset);
/*!
* extracts a grp meta item from a cursor at its
* current position
*/
RsGxsGrpMetaData* locked_getGrpMeta(RetroCursor& c);
RsGxsGrpMetaData* locked_getGrpMeta(RetroCursor& c, int colOffset);
/*!
* extracts a msg item from a cursor at its
@ -257,10 +258,12 @@ private:
std::list<std::string> mMsgColumns;
std::list<std::string> mMsgMetaColumns;
std::list<std::string> mMsgColumnsWithMeta;
std::list<std::string> mMsgIdColumn;
std::list<std::string> mGrpColumns;
std::list<std::string> mGrpMetaColumns;
std::list<std::string> mGrpColumnsWithMeta;
std::list<std::string> mGrpIdColumn;
// Message meta column
@ -287,6 +290,9 @@ private:
int mColMsg_MetaData;
int mColMsg_MsgId;
// Message columns with meta
int mColMsg_WithMetaOffset;
// Group meta columns
int mColGrpMeta_GrpId;
int mColGrpMeta_TimeStamp;
@ -319,6 +325,9 @@ private:
int mColGrp_NxsData;
int mColGrp_MetaData;
// Group columns with meta
int mColGrp_WithMetaOffset;
// Group id columns
int mColGrpId_GrpId;

File diff suppressed because it is too large Load diff

View file

@ -739,12 +739,13 @@ private:
/*!
* Attempts to validate msg signatures
* @param msg message to be validated
* @param grpFlag the flag for the group the message belongs to
* @param grpFlag the distribution flag for the group the message belongs to
* @param grpFlag the signature flag for the group the message belongs to
* @param grpKeySet the key set user has for the message's group
* @return VALIDATE_SUCCESS for success, VALIDATE_FAIL for fail,
* VALIDATE_ID_SIGN_NOT_AVAIL for Id sign key not avail (but requested)
*/
int validateMsg(RsNxsMsg* msg, const uint32_t& grpFlag, RsTlvSecurityKeySet& grpKeySet);
int validateMsg(RsNxsMsg* msg, const uint32_t& grpFlag, const uint32_t &signFlag, RsTlvSecurityKeySet& grpKeySet);
/*!
* Attempts to validate group signatures
@ -862,7 +863,7 @@ private:
std::vector<GroupDeletePublish> mGroupDeletePublish;
std::map<RsGxsId,std::set<RsPeerId> > mRoutingClues ;
std::list<std::pair<RsGxsMessageId,RsPeerId> > mTrackingClues ;
};
#endif // RSGENEXCHANGE_H

View file

@ -162,6 +162,7 @@ public:
*/
virtual bool getKey(const RsGxsId &id, RsTlvSecurityKey &key) = 0;
virtual bool getPrivateKey(const RsGxsId &id, RsTlvSecurityKey &key) = 0; // For signing outgoing messages.
virtual bool getIdDetails(const RsGxsId& id, RsIdentityDetails& details) = 0 ; // Proxy function so that we get p3Identity info from Gxs
};
class GixsReputation

View file

@ -33,7 +33,7 @@ RsGxsGrpMetaData::RsGxsGrpMetaData()
clear();
}
uint32_t RsGxsGrpMetaData::serial_size()
uint32_t RsGxsGrpMetaData::serial_size(uint32_t api_version)
{
uint32_t s = 8; // header size
@ -46,10 +46,15 @@ uint32_t RsGxsGrpMetaData::serial_size()
s += GetTlvStringSize(mServiceString);
s += signSet.TlvSize();
s += keys.TlvSize();
s += 4; // for mCircleType
s += 4; // for mCircleType
s += mCircleId.serial_size();
s += 4; // mAuthenFlag
s += mParentGrpId.serial_size();
s += 4; // mAuthenFlag
s += mParentGrpId.serial_size(); // mParentGroupId
if(api_version == RS_GXS_GRP_META_DATA_VERSION_ID_0002)
s += 4; // mSignFlag
else if(api_version != RS_GXS_GRP_META_DATA_VERSION_ID_0001)
std::cerr << "(EE) wrong/unknown API version " << api_version << " requested in RsGxsGrpMetaData::serial_size()" << std::endl;
return s;
}
@ -91,10 +96,9 @@ void RsGxsGrpMetaData::clear(){
mHash.clear() ;
}
bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize)
bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize,uint32_t api_version)
{
uint32_t tlvsize = serial_size() ;
uint32_t tlvsize = serial_size(api_version) ;
uint32_t offset = 0;
if (pktsize < tlvsize)
@ -104,7 +108,7 @@ bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize)
bool ok = true;
ok &= setRsItemHeader(data, tlvsize, 0, tlvsize);
ok &= setRsItemHeader(data, tlvsize, api_version, tlvsize);
#ifdef GXS_DEBUG
std::cerr << "RsGxsGrpMetaData serialise()" << std::endl;
@ -130,6 +134,8 @@ bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize)
ok &= signSet.SetTlv(data, tlvsize, &offset);
ok &= keys.SetTlv(data, tlvsize, &offset);
if(api_version == RS_GXS_GRP_META_DATA_VERSION_ID_0002)
ok &= setRawUInt32(data, tlvsize, &offset, mSignFlags); // new in API v2. Was previously missing. Kept in the end for backward compatibility
return ok;
}
@ -154,12 +160,31 @@ bool RsGxsGrpMetaData::deserialise(void *data, uint32_t &pktsize)
ok &= getRawUInt32(data, pktsize, &offset, &mPublishTs);
ok &= getRawUInt32(data, pktsize, &offset, &mCircleType);
ok &= getRawUInt32(data, pktsize, &offset, &mAuthenFlags);
ok &= mAuthorId.deserialise(data, pktsize, offset);
ok &= GetTlvString(data, pktsize, &offset, 0, mServiceString);
ok &= mCircleId.deserialise(data, pktsize, offset);
ok &= signSet.GetTlv(data, pktsize, &offset);
ok &= keys.GetTlv(data, pktsize, &offset);
switch(getRsItemId(data))
{
case RS_GXS_GRP_META_DATA_VERSION_ID_0002: ok &= getRawUInt32(data, pktsize, &offset, &mSignFlags); // current API
break ;
case RS_GXS_GRP_META_DATA_VERSION_ID_0001: mSignFlags = 0; // old API. Do not leave this uninitialised!
break ;
default:
std::cerr << "(EE) RsGxsGrpMetaData::deserialise(): ERROR: unknown API version " << std::hex << getRsItemId(data) << std::dec << std::endl;
}
if(offset != pktsize)
{
std::cerr << "(EE) RsGxsGrpMetaData::deserialise(): ERROR: unmatched size " << offset << ", expected: " << pktsize << std::dec << std::endl;
return false ;
}
#ifdef DROP_NON_CANONICAL_ITEMS
if(mGroupName.length() > RsGxsGrpMetaData::MAX_ALLOWED_STRING_SIZE)
{
@ -202,8 +227,8 @@ uint32_t RsGxsMsgMetaData::serial_size()
s += signSet.TlvSize();
s += GetTlvStringSize(mMsgName);
s += 4;
s += 4;
s += 4; // mPublishTS
s += 4; // mMsgFlags
return s;
}
@ -239,7 +264,7 @@ bool RsGxsMsgMetaData::serialise(void *data, uint32_t *size)
bool ok = true;
ok &= setRsItemHeader(data, tlvsize, 0, tlvsize);
ok &= setRsItemHeader(data, tlvsize, RS_GXS_MSG_META_DATA_VERSION_ID_0002, tlvsize);
#ifdef GXS_DEBUG
std::cerr << "RsGxsGrpMetaData serialise()" << std::endl;

View file

@ -37,6 +37,13 @@
class RsGroupMetaData;
class RsMsgMetaData;
static const uint32_t RS_GXS_GRP_META_DATA_VERSION_ID_0001 = 0x0000 ; // change this, and keep old values if the content changes
static const uint32_t RS_GXS_GRP_META_DATA_VERSION_ID_0002 = 0xaf01 ; // current API
static const uint32_t RS_GXS_MSG_META_DATA_VERSION_ID_0002 = 0x0000 ; // current API
static const uint32_t RS_GXS_GRP_META_DATA_CURRENT_API_VERSION = RS_GXS_GRP_META_DATA_VERSION_ID_0002;
class RsGxsGrpMetaData
{
public:
@ -44,15 +51,15 @@ public:
RsGxsGrpMetaData();
bool deserialise(void *data, uint32_t &pktsize);
bool serialise(void* data, uint32_t &pktsize);
uint32_t serial_size();
bool serialise(void* data, uint32_t &pktsize, uint32_t api_version);
uint32_t serial_size(uint32_t api_version);
void clear();
void operator =(const RsGroupMetaData& rMeta);
RsGxsGroupId mGroupId;
RsGxsGroupId mOrigGrpId;
std::string mGroupName;
uint32_t mGroupFlags;
uint32_t mGroupFlags; // GXS_SERV::FLAG_PRIVACY_RESTRICTED | GXS_SERV::FLAG_PRIVACY_PRIVATE | GXS_SERV::FLAG_PRIVACY_PUBLIC
uint32_t mPublishTs;
uint32_t mSignFlags;
RsGxsId mAuthorId;
@ -123,7 +130,6 @@ public:
uint32_t recvTS;
RsFileHash mHash;
bool validated;
};

View file

@ -448,7 +448,7 @@ bool RsGxsDataAccess::getGroupData(const uint32_t& token, std::list<RsNxsGrp*>&
if(gmreq)
{
grpData = gmreq->mGroupData;
grpData.swap(gmreq->mGroupData);
gmreq->mGroupData.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
}else{
@ -480,7 +480,7 @@ bool RsGxsDataAccess::getMsgData(const uint32_t& token, NxsMsgDataResult& msgDat
if(mdreq)
{
msgData = mdreq->mMsgData;
msgData.swap(mdreq->mMsgData);
mdreq->mMsgData.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
}
@ -517,7 +517,7 @@ bool RsGxsDataAccess::getMsgRelatedData(const uint32_t &token, NxsMsgRelatedData
if(mrireq)
{
msgData = mrireq->mMsgDataResult;
msgData.swap(mrireq->mMsgDataResult);
mrireq->mMsgDataResult.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
}
@ -551,7 +551,7 @@ bool RsGxsDataAccess::getMsgSummary(const uint32_t& token, GxsMsgMetaResult& msg
if(mmreq)
{
msgInfo = mmreq->mMsgMetaData;
msgInfo.swap(mmreq->mMsgMetaData);
mmreq->mMsgMetaData.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
@ -591,7 +591,7 @@ bool RsGxsDataAccess::getMsgRelatedSummary(const uint32_t &token, MsgRelatedMeta
if(mrireq)
{
msgMeta = mrireq->mMsgMetaResult;
msgMeta.swap(mrireq->mMsgMetaResult);
mrireq->mMsgMetaResult.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
}
@ -630,7 +630,7 @@ bool RsGxsDataAccess::getMsgRelatedList(const uint32_t &token, MsgRelatedIdResul
if(mrireq)
{
msgIds = mrireq->mMsgIdResult;
msgIds.swap(mrireq->mMsgIdResult);
mrireq->mMsgIdResult.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
}
@ -664,7 +664,7 @@ bool RsGxsDataAccess::getMsgList(const uint32_t& token, GxsMsgIdResult& msgIds)
if(mireq)
{
msgIds = mireq->mMsgIdResult;
msgIds.swap(mireq->mMsgIdResult);
mireq->mMsgIdResult.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
}
@ -697,7 +697,7 @@ bool RsGxsDataAccess::getGroupList(const uint32_t& token, std::list<RsGxsGroupId
if(gireq)
{
groupIds = gireq->mGroupIdResult;
groupIds.swap(gireq->mGroupIdResult);
gireq->mGroupIdResult.clear();
locked_updateRequestStatus(token, GXS_REQUEST_V2_STATUS_DONE);
@ -1032,31 +1032,25 @@ bool RsGxsDataAccess::getGroupList(const std::list<RsGxsGroupId>& grpIdsIn, cons
bool RsGxsDataAccess::getMsgData(MsgDataReq* req)
{
GxsMsgResult result;
GxsMsgReq msgIdOut;
// filter based on options
getMsgList(req->mMsgIds, req->Options, msgIdOut);
mDataStore->retrieveNxsMsgs(msgIdOut, result, true, true);
mDataStore->retrieveNxsMsgs(msgIdOut, req->mMsgData, true, true);
req->mMsgData = result;
return true;
}
bool RsGxsDataAccess::getMsgSummary(MsgMetaReq* req)
{
GxsMsgMetaResult result;
GxsMsgReq msgIdOut;
// filter based on options
getMsgList(req->mMsgIds, req->Options, msgIdOut);
mDataStore->retrieveGxsMsgMetaData(msgIdOut, result);
req->mMsgMetaData = result;
mDataStore->retrieveGxsMsgMetaData(msgIdOut, req->mMsgMetaData);
return true;
}
@ -1558,7 +1552,7 @@ bool RsGxsDataAccess::getGroupStatistic(GroupStatisticRequest *req)
req->mGroupStatistic.mNumChildMsgsNew = 0;
req->mGroupStatistic.mNumChildMsgsUnread = 0;
for(int i = 0; i < msgMetaV.size(); ++i)
for(uint32_t i = 0; i < msgMetaV.size(); ++i)
{
RsGxsMsgMetaData* m = msgMetaV[i];
req->mGroupStatistic.mTotalSizeOfMsgs += m->mMsgSize + m->serial_size();
@ -1609,7 +1603,7 @@ bool RsGxsDataAccess::getServiceStatistic(ServiceStatisticRequest *req)
for(; mit != grpMeta.end(); ++mit)
{
RsGxsGrpMetaData* m = mit->second;
req->mServiceStatistic.mSizeOfGrps += m->mGrpSize + m->serial_size();
req->mServiceStatistic.mSizeOfGrps += m->mGrpSize + m->serial_size(RS_GXS_GRP_META_DATA_CURRENT_API_VERSION);
if (IS_GROUP_SUBSCRIBED(m->mSubscribeFlags))
{

File diff suppressed because it is too large Load diff

File diff suppressed because it is too large Load diff

View file

@ -99,7 +99,7 @@ bool MsgRespPending::accepted()
GixsReputation rep;
if(getAuthorRep(rep, entry.mAuthorId, mPeerId))
{
if(rep.score > mCutOff)
if(rep.score >= mCutOff)
{
entry.mPassedVetting = true;
count++;
@ -134,7 +134,7 @@ bool GrpRespPending::accepted()
if(getAuthorRep(rep, entry.mAuthorId, mPeerId))
{
if(rep.score > mCutOff)
if(rep.score >= mCutOff)
{
entry.mPassedVetting = true;
count++;

View file

@ -27,8 +27,13 @@
#include "rsgxsutil.h"
#include "retroshare/rsgxsflags.h"
#include "retroshare/rspeers.h"
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!)
//#define GXSUTIL_DEBUG 1
RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, uint32_t messageStorePeriod, uint32_t chunkSize)
: mDs(dataService), MESSAGE_STORE_PERIOD(messageStorePeriod), CHUNK_SIZE(chunkSize)
@ -106,9 +111,8 @@ bool RsGxsMessageCleanUp::clean()
return mGrpMeta.empty();
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService) :
mDs(dataService), mDone(false), mIntegrityMutex("integrity")
RsGxsIntegrityCheck::RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs) :
mDs(dataService), mDone(false), mIntegrityMutex("integrity"),mGixs(gixs)
{ }
void RsGxsIntegrityCheck::run()
@ -118,122 +122,206 @@ void RsGxsIntegrityCheck::run()
bool RsGxsIntegrityCheck::check()
{
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
mDs->retrieveNxsGrps(grp, true, true);
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgIds;
GxsMsgReq grps;
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
mDs->retrieveNxsGrps(grp, true, true);
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgIds;
GxsMsgReq grps;
std::set<RsGxsId> used_gxs_ids ;
std::set<RsGxsGroupId> subscribed_groups ;
// compute hash and compare to stored value, if it fails then simply add it
// to list
std::map<RsGxsGroupId, RsNxsGrp*>::iterator git = grp.begin();
for(; git != grp.end(); ++git)
{
RsNxsGrp* grp = git->second;
RsFileHash currHash;
pqihash pHash;
pHash.addData(grp->grp.bin_data, grp->grp.bin_len);
pHash.Complete(currHash);
// compute hash and compare to stored value, if it fails then simply add it
// to list
std::map<RsGxsGroupId, RsNxsGrp*>::iterator git = grp.begin();
for(; git != grp.end(); ++git)
{
RsNxsGrp* grp = git->second;
RsFileHash currHash;
pqihash pHash;
pHash.addData(grp->grp.bin_data, grp->grp.bin_len);
pHash.Complete(currHash);
if(currHash == grp->metaData->mHash)
{
// get all message ids of group
if (mDs->retrieveMsgIds(grp->grpId, msgIds[grp->grpId]) == 1)
{
// store the group for retrieveNxsMsgs
grps[grp->grpId];
}
else
{
msgIds.erase(msgIds.find(grp->grpId));
// grpsToDel.push_back(grp->grpId);
}
}
else
{
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
if(currHash == grp->metaData->mHash)
{
// get all message ids of group
if (mDs->retrieveMsgIds(grp->grpId, msgIds[grp->grpId]) == 1)
{
// store the group for retrieveNxsMsgs
grps[grp->grpId];
mDs->removeGroups(grpsToDel);
if(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)
{
subscribed_groups.insert(git->first) ;
// now messages
GxsMsgReq msgsToDel;
GxsMsgResult msgs;
if(!grp->metaData->mAuthorId.isNull())
{
#ifdef GXSUTIL_DEBUG
std::cerr << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl;
#endif
mDs->retrieveNxsMsgs(grps, msgs, false, true);
used_gxs_ids.insert(grp->metaData->mAuthorId) ;
}
}
}
else
{
msgIds.erase(msgIds.find(grp->grpId));
// grpsToDel.push_back(grp->grpId);
}
// check msg ids and messages
GxsMsgReq::iterator msgIdsIt;
for (msgIdsIt = msgIds.begin(); msgIdsIt != msgIds.end(); ++msgIdsIt)
{
const RsGxsGroupId& grpId = msgIdsIt->first;
std::vector<RsGxsMessageId> &msgIdV = msgIdsIt->second;
}
else
{
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
std::vector<RsGxsMessageId>::iterator msgIdIt;
for (msgIdIt = msgIdV.begin(); msgIdIt != msgIdV.end(); ++msgIdIt)
{
const RsGxsMessageId& msgId = *msgIdIt;
std::vector<RsNxsMsg*> &nxsMsgV = msgs[grpId];
mDs->removeGroups(grpsToDel);
std::vector<RsNxsMsg*>::iterator nxsMsgIt;
for (nxsMsgIt = nxsMsgV.begin(); nxsMsgIt != nxsMsgV.end(); ++nxsMsgIt)
{
RsNxsMsg *nxsMsg = *nxsMsgIt;
if (nxsMsg && msgId == nxsMsg->msgId)
{
break;
}
}
// now messages
GxsMsgReq msgsToDel;
GxsMsgResult msgs;
if (nxsMsgIt == nxsMsgV.end())
{
msgsToDel[grpId].push_back(msgId);
}
}
}
mDs->retrieveNxsMsgs(grps, msgs, false, true);
GxsMsgResult::iterator mit = msgs.begin();
// check msg ids and messages
GxsMsgReq::iterator msgIdsIt;
for (msgIdsIt = msgIds.begin(); msgIdsIt != msgIds.end(); ++msgIdsIt)
{
const RsGxsGroupId& grpId = msgIdsIt->first;
std::vector<RsGxsMessageId> &msgIdV = msgIdsIt->second;
for(; mit != msgs.end(); ++mit)
{
std::vector<RsNxsMsg*>& msgV = mit->second;
std::vector<RsNxsMsg*>::iterator vit = msgV.begin();
std::vector<RsGxsMessageId>::iterator msgIdIt;
for (msgIdIt = msgIdV.begin(); msgIdIt != msgIdV.end(); ++msgIdIt)
{
const RsGxsMessageId& msgId = *msgIdIt;
std::vector<RsNxsMsg*> &nxsMsgV = msgs[grpId];
for(; vit != msgV.end(); ++vit)
{
RsNxsMsg* msg = *vit;
RsFileHash currHash;
pqihash pHash;
pHash.addData(msg->msg.bin_data, msg->msg.bin_len);
pHash.Complete(currHash);
std::vector<RsNxsMsg*>::iterator nxsMsgIt;
for (nxsMsgIt = nxsMsgV.begin(); nxsMsgIt != nxsMsgV.end(); ++nxsMsgIt)
{
RsNxsMsg *nxsMsg = *nxsMsgIt;
if (nxsMsg && msgId == nxsMsg->msgId)
{
break;
}
}
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << "(EE) deleting message data with wrong hash or null meta data. meta=" << (void*)msg->metaData << std::endl;
msgsToDel[msg->grpId].push_back(msg->msgId);
}
if (nxsMsgIt == nxsMsgV.end())
{
msgsToDel[grpId].push_back(msgId);
}
}
}
delete msg;
}
}
GxsMsgResult::iterator mit = msgs.begin();
mDs->removeMsgs(msgsToDel);
for(; mit != msgs.end(); ++mit)
{
std::vector<RsNxsMsg*>& msgV = mit->second;
std::vector<RsNxsMsg*>::iterator vit = msgV.begin();
RsStackMutex stack(mIntegrityMutex);
mDone = true;
for(; vit != msgV.end(); ++vit)
{
RsNxsMsg* msg = *vit;
RsFileHash currHash;
pqihash pHash;
pHash.addData(msg->msg.bin_data, msg->msg.bin_len);
pHash.Complete(currHash);
std::vector<RsGxsGroupId>::iterator grpIt;
for(grpIt = grpsToDel.begin(); grpIt != grpsToDel.end(); ++grpIt)
{
mDeletedGrps.push_back(*grpIt);
}
mDeletedMsgs = msgsToDel;
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << "(EE) deleting message data with wrong hash or null meta data. meta=" << (void*)msg->metaData << std::endl;
msgsToDel[msg->grpId].push_back(msg->msgId);
}
else if(!msg->metaData->mAuthorId.isNull() && subscribed_groups.find(msg->metaData->mGroupId)!=subscribed_groups.end())
{
#ifdef GXSUTIL_DEBUG
std::cerr << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl;
#endif
used_gxs_ids.insert(msg->metaData->mAuthorId) ;
}
return true;
delete msg;
}
}
mDs->removeMsgs(msgsToDel);
RsStackMutex stack(mIntegrityMutex);
mDone = true;
std::vector<RsGxsGroupId>::iterator grpIt;
for(grpIt = grpsToDel.begin(); grpIt != grpsToDel.end(); ++grpIt)
{
mDeletedGrps.push_back(*grpIt);
}
mDeletedMsgs = msgsToDel;
#ifdef GXSUTIL_DEBUG
std::cerr << "At end of pass, this is the list used GXS ids: " << std::endl;
std::cerr << " requesting them to GXS identity service to enforce loading." << std::endl;
#endif
std::list<RsPeerId> connected_friends ;
rsPeers->getOnlineList(connected_friends) ;
std::vector<RsGxsId> gxs_ids ;
for(std::set<RsGxsId>::const_iterator it(used_gxs_ids.begin());it!=used_gxs_ids.end();++it)
{
gxs_ids.push_back(*it) ;
#ifdef GXSUTIL_DEBUG
std::cerr << " " << *it << std::endl;
#endif
}
int nb_requested_not_in_cache = 0;
#ifdef GXSUTIL_DEBUG
std::cerr << " issuing random get on friends for non existing IDs" << std::endl;
#endif
// now request a cache update for them, which triggers downloading from friends, if missing.
for(;nb_requested_not_in_cache<MAX_GXS_IDS_REQUESTS_NET && gxs_ids.size()>0;)
{
uint32_t n = RSRandom::random_u32() % gxs_ids.size() ;
#ifdef GXSUTIL_DEBUG
std::cerr << " requesting ID " << gxs_ids[n] ;
#endif
if(!mGixs->haveKey(gxs_ids[n])) // checks if we have it already in the cache (conservative way to ensure that we atually have it)
{
mGixs->requestKey(gxs_ids[n],connected_friends);
++nb_requested_not_in_cache ;
#ifdef GXSUTIL_DEBUG
std::cerr << " ... from cache/net" << std::endl;
#endif
}
else
{
#ifdef GXSUTIL_DEBUG
std::cerr << " ... already in cache" << std::endl;
#endif
// Note: we could time_stamp even in the case where the id is not cached. Anyway, it's not really a problem here, since IDs have a high chance of
// behing eventually stamped.
mGixs->timeStampKey(gxs_ids[n]) ;
}
gxs_ids[n] = gxs_ids[gxs_ids.size()-1] ;
gxs_ids.pop_back() ;
}
#ifdef GXSUTIL_DEBUG
std::cerr << " total actual cache requests: "<< nb_requested_not_in_cache << std::endl;
#endif
return true;
}
bool RsGxsIntegrityCheck::isDone()

View file

@ -30,6 +30,8 @@
#include "serialiser/rsnxsitems.h"
#include "rsgds.h"
class RsGixs ;
/*!
* Handy function for cleaning out meta result containers
* @param container
@ -40,10 +42,9 @@ void freeAndClearContainerResource(Container container)
typename Container::iterator meta_it = container.begin();
for(; meta_it != container.end(); ++meta_it)
{
delete meta_it->second;
if(meta_it->second != NULL)
delete meta_it->second;
}
container.clear();
}
@ -112,7 +113,7 @@ public:
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck(RsGeneralDataService* const dataService);
RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs);
bool check();
@ -129,6 +130,8 @@ private:
RsMutex mIntegrityMutex;
std::list<RsGxsGroupId> mDeletedGrps;
std::map<RsGxsGroupId, std::vector<RsGxsMessageId> > mDeletedMsgs;
RsGixs *mGixs ;
};
class GroupUpdate

View file

@ -127,6 +127,13 @@ public:
*/
virtual int sharePublishKey(const RsGxsGroupId& grpId,const std::set<RsPeerId>& peers)=0 ;
/*!
* \brief rejectMessage
* Tells the network exchange service to not download this message again, at least for some time (maybe 24h or more)
* in order to avoid cluttering the network pipe with copied of this rejected message.
* \param msgId
*/
virtual void rejectMessage(const RsGxsMessageId& msgId) =0;
};
#endif // RSGNP_H