Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #208 from csoler/v0.6-GxsDebug

Browse Source 
V0.6 gxs debug
This commit is contained in:
Cyril Soler 2015-12-17 09:23:16 -05:00
commit 45fb453e92
10 changed files with 1293 additions and 846 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

164
libretroshare/src/gxs/gxssecurity.cc
View File

@ -27,6 +27,7 @@
#include "gxssecurity.h"
#include "pqi/authgpg.h"
#include "util/rsdir.h"
#include "util/rsmemory.h"
//#include "retroshare/rspeers.h"
/****
@ -607,117 +608,114 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in,
bool GxsSecurity::validateNxsGrp(const RsNxsGrp& grp, const RsTlvKeySignature& sign, const RsTlvSecurityKey& key)
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << std::endl;
std::cerr << "RsNxsGrp :";
std::cerr << std::endl;
grp.print(std::cerr, 10);
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << std::endl;
std::cerr << "RsNxsGrp :";
std::cerr << std::endl;
grp.print(std::cerr, 10);
std::cerr << std::endl;
#endif
RsGxsGrpMetaData& grpMeta = *(grp.metaData);
RsGxsGrpMetaData& grpMeta = *(grp.metaData);
/********************* check signature *******************/
/********************* check signature *******************/
/* check signature timeperiod */
if ((grpMeta.mPublishTs < key.startTS) || (key.endTS != 0 && grpMeta.mPublishTs > key.endTS))
{
/* check signature timeperiod */
if ((grpMeta.mPublishTs < key.startTS) || (key.endTS != 0 && grpMeta.mPublishTs > key.endTS))
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << " GxsSecurity::validateNxsMsg() TS out of range";
std::cerr << std::endl;
std::cerr << " GxsSecurity::validateNxsMsg() TS out of range";
std::cerr << std::endl;
#endif
return false;
}
return false;
}
/* decode key */
const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data;
long keylen = key.keyData.bin_len;
unsigned int siglen = sign.signData.bin_len;
unsigned char *sigbuf = (unsigned char *) sign.signData.bin_data;
/* decode key */
const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data;
long keylen = key.keyData.bin_len;
unsigned int siglen = sign.signData.bin_len;
unsigned char *sigbuf = (unsigned char *) sign.signData.bin_data;
#ifdef DISTRIB_DEBUG
std::cerr << "GxsSecurity::validateNxsMsg() Decode Key";
std::cerr << " keylen: " << keylen << " siglen: " << siglen;
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsMsg() Decode Key";
std::cerr << " keylen: " << keylen << " siglen: " << siglen;
std::cerr << std::endl;
#endif
/* extract admin key */
RSA *rsakey = (key.keyFlags & RSTLV_KEY_TYPE_FULL)? d2i_RSAPrivateKey(NULL, &(keyptr), keylen): d2i_RSAPublicKey(NULL, &(keyptr), keylen);
/* extract admin key */
RSA *rsakey = (key.keyFlags & RSTLV_KEY_TYPE_FULL)? d2i_RSAPrivateKey(NULL, &(keyptr), keylen): d2i_RSAPublicKey(NULL, &(keyptr), keylen);
if (!rsakey)
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << " Invalid RSA Key";
std::cerr << std::endl;
key.print(std::cerr, 10);
#endif
}
RsTlvKeySignatureSet signSet = grpMeta.signSet;
grpMeta.signSet.TlvClear();
uint32_t metaDataLen = grpMeta.serial_size();
uint32_t allGrpDataLen = metaDataLen + grp.grp.bin_len;
char* metaData = new char[metaDataLen];
char* allGrpData = new char[allGrpDataLen]; // msgData + metaData
grpMeta.serialise(metaData, metaDataLen);
// copy msg data and meta in allmsgData buffer
memcpy(allGrpData, grp.grp.bin_data, grp.grp.bin_len);
memcpy(allGrpData+(grp.grp.bin_len), metaData, metaDataLen);
delete[] metaData ;
EVP_PKEY *signKey = EVP_PKEY_new();
EVP_PKEY_assign_RSA(signKey, rsakey);
/* calc and check signature */
EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
EVP_VerifyInit(mdctx, EVP_sha1());
EVP_VerifyUpdate(mdctx, allGrpData, allGrpDataLen);
int signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey);
EVP_MD_CTX_destroy(mdctx);
if(signOk != 1) // try previous API. This is a hack to accept groups previously signed with old APIs.
if (!rsakey)
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp()";
std::cerr << " Invalid RSA Key";
std::cerr << std::endl;
key.print(std::cerr, 10);
#endif
}
std::vector<uint32_t> api_versions_to_check ;
api_versions_to_check.push_back(RS_GXS_GRP_META_DATA_VERSION_ID_0002) ; // put newest first, for debug info purpose
api_versions_to_check.push_back(RS_GXS_GRP_META_DATA_VERSION_ID_0001) ;
RsTlvKeySignatureSet signSet = grpMeta.signSet;
grpMeta.signSet.TlvClear();
int signOk =0;
EVP_PKEY *signKey = EVP_PKEY_new();
EVP_PKEY_assign_RSA(signKey, rsakey);
for(uint32_t i=0;i<api_versions_to_check.size() && 0==signOk;++i)
{
uint32_t metaDataLen = grpMeta.serial_size(api_versions_to_check[i]);
uint32_t allGrpDataLen = metaDataLen + grp.grp.bin_len;
RsTemporaryMemory metaData(metaDataLen) ;
RsTemporaryMemory allGrpData(allGrpDataLen) ;// msgData + metaData
grpMeta.serialise(metaData, metaDataLen,api_versions_to_check[i]);
// copy msg data and meta in allmsgData buffer
memcpy(allGrpData, grp.grp.bin_data, grp.grp.bin_len);
memcpy(allGrpData+(grp.grp.bin_len), metaData, metaDataLen);
/* calc and check signature */
EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
EVP_VerifyInit(mdctx, EVP_sha1());
EVP_VerifyUpdate(mdctx, allGrpData, allGrpDataLen-4); // that means ommit the last
EVP_VerifyUpdate(mdctx, allGrpData, allGrpDataLen);
signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey);
EVP_MD_CTX_destroy(mdctx);
if(signOk)
std::cerr << "(WW) GXS group with old API found. Signature still checks!" << std::endl;
if(i>0)
std::cerr << "(WW) Checking group signature with old api version " << i+1 << " : tag " << std::hex << api_versions_to_check[i] << std::dec << " result: " << signOk << std::endl;
}
delete[] allGrpData ;
/* clean up */
EVP_PKEY_free(signKey);
/* clean up */
EVP_PKEY_free(signKey);
grpMeta.signSet = signSet;
// restore data
if (signOk == 1)
{
grpMeta.signSet = signSet;
if (signOk == 1)
{
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp() Signature OK";
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsGrp() Signature OK";
std::cerr << std::endl;
#endif
return true;
}
return true;
}
#ifdef GXS_SECURITY_DEBUG
std::cerr << "GxsSecurity::validateNxsGrp() Signature invalid";
std::cerr << std::endl;
std::cerr << "GxsSecurity::validateNxsGrp() Signature invalid";
std::cerr << std::endl;
#endif
return false;
return false;
}

184
libretroshare/src/gxs/rsgenexchange.cc
View File

@ -77,10 +77,10 @@ RsGenExchange::RsGenExchange(RsGeneralDataService *gds, RsNetworkExchangeService
mAuthenPolicy(authenPolicy),
MESSAGE_STORE_PERIOD(messageStorePeriod),
mCleaning(false),
mLastClean(time(NULL)),
mLastClean((int)time(NULL) - (int)(RSRandom::random_u32() % MSG_CLEANUP_PERIOD)), // this helps unsynchronising the checks for the different services
mMsgCleanUp(NULL),
mChecking(false),
mLastCheck(time(NULL)),
mLastCheck((int)time(NULL) - (int)(RSRandom::random_u32() % INTEGRITY_CHECK_PERIOD)), // this helps unsynchronising the checks for the different services
mIntegrityCheck(NULL),
CREATE_FAIL(0),
CREATE_SUCCESS(1),
@ -196,7 +196,7 @@ void RsGenExchange::tick()
service_tick();
time_t now = time(NULL);
if((mLastClean + MSG_CLEANUP_PERIOD < now) || mCleaning)
{
if(mMsgCleanUp)
@ -252,7 +252,7 @@ void RsGenExchange::tick()
}
else
{
mIntegrityCheck = new RsGxsIntegrityCheck(mDataStore);
mIntegrityCheck = new RsGxsIntegrityCheck(mDataStore,mGixs);
mIntegrityCheck->start();
mChecking = true;
}
@ -410,12 +410,12 @@ uint8_t RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& privateKe
// group is self signing
// for the creation of group signature
// only public admin and publish keys are present in meta
uint32_t metaDataLen = meta->serial_size();
uint32_t metaDataLen = meta->serial_size(RS_GXS_GRP_META_DATA_CURRENT_API_VERSION);
uint32_t allGrpDataLen = metaDataLen + grp->grp.bin_len;
char* metaData = new char[metaDataLen];
char* allGrpData = new char[allGrpDataLen]; // msgData + metaData
meta->serialise(metaData, metaDataLen);
meta->serialise(metaData, metaDataLen,RS_GXS_GRP_META_DATA_CURRENT_API_VERSION);
// copy msg data and meta in allMsgData buffer
memcpy(allGrpData, grp->grp.bin_data, grp->grp.bin_len);
@ -499,13 +499,11 @@ int RsGenExchange::createGroupSignatures(RsTlvKeySignatureSet& signSet, RsTlvBin
authorKey, sign))
{
id_ret = SIGN_SUCCESS;
mGixs->timeStampKey(grpMeta.mAuthorId) ;
signSet.keySignSet[INDEX_AUTHEN_IDENTITY] = sign;
}
else
{
id_ret = SIGN_FAIL;
}
signSet.keySignSet[INDEX_AUTHEN_IDENTITY] = sign;
}
else
{
@ -661,23 +659,20 @@ int RsGenExchange::createMsgSignatures(RsTlvKeySignatureSet& signSet, RsTlvBinar
bool haveKey = mGixs->havePrivateKey(msgMeta.mAuthorId);
if(haveKey)
{
RsTlvSecurityKey authorKey;
mGixs->getPrivateKey(msgMeta.mAuthorId, authorKey);
RsTlvKeySignature sign;
{
RsTlvSecurityKey authorKey;
mGixs->getPrivateKey(msgMeta.mAuthorId, authorKey);
RsTlvKeySignature sign;
if(GxsSecurity::getSignature((char*)msgData.bin_data, msgData.bin_len,
authorKey, sign))
{
id_ret = SIGN_SUCCESS;
}
else
{
id_ret = SIGN_FAIL;
}
signSet.keySignSet[INDEX_AUTHEN_IDENTITY] = sign;
}
if(GxsSecurity::getSignature((char*)msgData.bin_data, msgData.bin_len, authorKey, sign))
{
id_ret = SIGN_SUCCESS;
mGixs->timeStampKey(msgMeta.mAuthorId) ;
signSet.keySignSet[INDEX_AUTHEN_IDENTITY] = sign;
}
else
id_ret = SIGN_FAIL;
}
else
{
mGixs->requestPrivateKey(msgMeta.mAuthorId);
@ -970,77 +965,88 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, const uin
int RsGenExchange::validateGrp(RsNxsGrp* grp)
{
bool needIdentitySign = false, idValidate = false;
RsGxsGrpMetaData& metaData = *(grp->metaData);
bool needIdentitySign = false, idValidate = false;
RsGxsGrpMetaData& metaData = *(grp->metaData);
uint8_t author_flag = GXS_SERV::GRP_OPTION_AUTHEN_AUTHOR_SIGN;
PrivacyBitPos pos = GRP_OPTION_BITS;
#ifdef GEN_EXCH_DEBUG
std::cerr << "Validating group " << grp->grpId << ", authorId = " << metaData.mAuthorId << std::endl;
#endif
// Check required permissions, and allow them to sign it - if they want too - as well!
if ((!metaData.mAuthorId.isNull()) || checkAuthenFlag(pos, author_flag))
{
needIdentitySign = true;
needIdentitySign = true;
#ifdef GEN_EXCH_DEBUG
std::cerr << "Needs Identity sign! (Service Flags)";
std::cerr << std::endl;
std::cerr << " Needs Identity sign! (Service Flags). Identity signing key is " << metaData.mAuthorId << std::endl;
#endif
}
if(needIdentitySign)
{
if(mGixs)
{
bool haveKey = mGixs->haveKey(metaData.mAuthorId);
if(needIdentitySign)
{
if(mGixs)
{
bool haveKey = mGixs->haveKey(metaData.mAuthorId);
if(haveKey)
{
RsTlvSecurityKey authorKey;
bool auth_key_fetched = mGixs->getKey(metaData.mAuthorId, authorKey) ;
if (auth_key_fetched)
{
RsTlvKeySignature sign = metaData.signSet.keySignSet[INDEX_AUTHEN_IDENTITY];
idValidate = GxsSecurity::validateNxsGrp(*grp, sign, authorKey);
mGixs->timeStampKey(metaData.mAuthorId) ;
}
else
{
std::cerr << "RsGenExchange::validateGrp()";
std::cerr << " ERROR Cannot Retrieve AUTHOR KEY for Group Sign Validation";
std::cerr << std::endl;
idValidate = false;
}
}else
{
std::list<RsPeerId> peers;
peers.push_back(grp->PeerId());
mGixs->requestKey(metaData.mAuthorId, peers);
return VALIDATE_FAIL_TRY_LATER;
}
}
else
{
if(haveKey)
{
#ifdef GEN_EXCH_DEBUG
std::cerr << "Gixs not enabled while request identity signature validation!" << std::endl;
std::cerr << " have ID key in cache: yes" << std::endl;
#endif
idValidate = false;
}
}
else
{
idValidate = true;
}
if(idValidate)
return VALIDATE_SUCCESS;
else
return VALIDATE_FAIL;
RsTlvSecurityKey authorKey;
bool auth_key_fetched = mGixs->getKey(metaData.mAuthorId, authorKey) ;
if (auth_key_fetched)
{
RsTlvKeySignature sign = metaData.signSet.keySignSet[INDEX_AUTHEN_IDENTITY];
idValidate = GxsSecurity::validateNxsGrp(*grp, sign, authorKey);
#ifdef GEN_EXCH_DEBUG
std::cerr << " key ID validation result: " << idValidate << std::endl;
#endif
mGixs->timeStampKey(metaData.mAuthorId) ;
}
else
{
std::cerr << "RsGenExchange::validateGrp()";
std::cerr << " ERROR Cannot Retrieve AUTHOR KEY for Group Sign Validation";
std::cerr << std::endl;
idValidate = false;
}
}else
{
#ifdef GEN_EXCH_DEBUG
std::cerr << " have key in cache: no. Return VALIDATE_LATER" << std::endl;
std::cerr << " requesting key " << metaData.mAuthorId << " to origin peer " << grp->PeerId() << std::endl;
#endif
std::list<RsPeerId> peers;
peers.push_back(grp->PeerId());
mGixs->requestKey(metaData.mAuthorId, peers);
return VALIDATE_FAIL_TRY_LATER;
}
}
else
{
#ifdef GEN_EXCH_DEBUG
std::cerr << " (EE) Gixs not enabled while request identity signature validation!" << std::endl;
#endif
idValidate = false;
}
}
else
{
idValidate = true;
}
if(idValidate)
return VALIDATE_SUCCESS;
else
return VALIDATE_FAIL;
}
@ -2337,9 +2343,9 @@ void RsGenExchange::publishGrps()
if(create == CREATE_SUCCESS)
{
uint32_t mdSize = grp->metaData->serial_size();
uint32_t mdSize = grp->metaData->serial_size(RS_GXS_GRP_META_DATA_CURRENT_API_VERSION);
char* metaData = new char[mdSize];
serialOk = grp->metaData->serialise(metaData, mdSize);
serialOk = grp->metaData->serialise(metaData, mdSize,RS_GXS_GRP_META_DATA_CURRENT_API_VERSION);
grp->meta.setBinData(metaData, mdSize);
delete[] metaData;
@ -2756,6 +2762,9 @@ void RsGenExchange::processRecvdGroups()
if(deserialOk)
{
#ifdef GEN_EXCH_DEBUG
std::cerr << " processing validation for group " << meta->mGroupId << ", attempts number " << gpsi.mAttempts << std::endl;
#endif
grp->metaData = meta;
uint8_t ret = validateGrp(grp);
@ -2770,7 +2779,7 @@ void RsGenExchange::processRecvdGroups()
// group has been validated. Let's notify the global router for the clue
#ifdef GEN_EXCH_DEBUG
std::cerr << "Group routage info: Identity=" << meta->mAuthorId << " from " << grp->PeerId() << std::endl;
std::cerr << " Group routage info: Identity=" << meta->mAuthorId << " from " << grp->PeerId() << std::endl;
#endif
if(!meta->mAuthorId.isNull())
@ -2797,7 +2806,7 @@ void RsGenExchange::processRecvdGroups()
else if(ret == VALIDATE_FAIL)
{
#ifdef GEN_EXCH_DEBUG
std::cerr << "failed to validate incoming meta, grpId: " << grp->grpId << ": wrong signature" << std::endl;
std::cerr << " failed to validate incoming meta, grpId: " << grp->grpId << ": wrong signature" << std::endl;
#endif
delete grp;
erase = true;
@ -2806,13 +2815,16 @@ void RsGenExchange::processRecvdGroups()
{
#ifdef GEN_EXCH_DEBUG
std::cerr << "failed to validate incoming grp, trying again. grpId: " << grp->grpId << std::endl;
std::cerr << " failed to validate incoming grp, trying again. grpId: " << grp->grpId << std::endl;
#endif
if(gpsi.mAttempts == VALIDATE_MAX_ATTEMPTS)
{
delete grp;
erase = true;
#ifdef GEN_EXCH_DEBUG
std::cerr << " max attempts " << VALIDATE_MAX_ATTEMPTS << " reached. Will delete group " << grp->grpId << std::endl;
#endif
}
else
{

25
libretroshare/src/gxs/rsgxsdata.cc
View File

@ -28,17 +28,12 @@
#include "serialiser/rsbaseserial.h"
#include "serialiser/rstlvbase.h"
static const uint32_t RS_GXS_GRP_META_DATA_VERSION_ID_0001 = 0x0000 ; // change this, and keep old values if the content changes
static const uint32_t RS_GXS_GRP_META_DATA_VERSION_ID_0002 = 0xaf01 ; // current API
static const uint32_t RS_GXS_MSG_META_DATA_VERSION_ID_0001 = 0x0000 ; // current API
RsGxsGrpMetaData::RsGxsGrpMetaData()
{
clear();
}
uint32_t RsGxsGrpMetaData::serial_size()
uint32_t RsGxsGrpMetaData::serial_size(uint32_t api_version)
{
uint32_t s = 8; // header size
@ -55,7 +50,11 @@ uint32_t RsGxsGrpMetaData::serial_size()
s += mCircleId.serial_size();
s += 4; // mAuthenFlag
s += mParentGrpId.serial_size(); // mParentGroupId
s += 4; // mSignFlag
if(api_version == RS_GXS_GRP_META_DATA_VERSION_ID_0002)
s += 4; // mSignFlag
else if(api_version != RS_GXS_GRP_META_DATA_VERSION_ID_0001)
std::cerr << "(EE) wrong/unknown API version " << api_version << " requested in RsGxsGrpMetaData::serial_size()" << std::endl;
return s;
}
@ -97,10 +96,9 @@ void RsGxsGrpMetaData::clear(){
mHash.clear() ;
}
bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize)
bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize,uint32_t api_version)
{
uint32_t tlvsize = serial_size() ;
uint32_t tlvsize = serial_size(api_version) ;
uint32_t offset = 0;
if (pktsize < tlvsize)
@ -110,7 +108,7 @@ bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize)
bool ok = true;
ok &= setRsItemHeader(data, tlvsize, RS_GXS_GRP_META_DATA_VERSION_ID_0002, tlvsize);
ok &= setRsItemHeader(data, tlvsize, api_version, tlvsize);
#ifdef GXS_DEBUG
std::cerr << "RsGxsGrpMetaData serialise()" << std::endl;
@ -136,7 +134,8 @@ bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize)
ok &= signSet.SetTlv(data, tlvsize, &offset);
ok &= keys.SetTlv(data, tlvsize, &offset);
ok &= setRawUInt32(data, tlvsize, &offset, mSignFlags); // new in API v2. Was previously missing. Kept in the end for backward compatibility
if(api_version == RS_GXS_GRP_META_DATA_VERSION_ID_0002)
ok &= setRawUInt32(data, tlvsize, &offset, mSignFlags); // new in API v2. Was previously missing. Kept in the end for backward compatibility
return ok;
}
@ -265,7 +264,7 @@ bool RsGxsMsgMetaData::serialise(void *data, uint32_t *size)
bool ok = true;
ok &= setRsItemHeader(data, tlvsize, RS_GXS_MSG_META_DATA_VERSION_ID_0001, tlvsize);
ok &= setRsItemHeader(data, tlvsize, RS_GXS_MSG_META_DATA_VERSION_ID_0002, tlvsize);
#ifdef GXS_DEBUG
std::cerr << "RsGxsGrpMetaData serialise()" << std::endl;

11
libretroshare/src/gxs/rsgxsdata.h
View File

@ -37,6 +37,13 @@
class RsGroupMetaData;
class RsMsgMetaData;
static const uint32_t RS_GXS_GRP_META_DATA_VERSION_ID_0001 = 0x0000 ; // change this, and keep old values if the content changes
static const uint32_t RS_GXS_GRP_META_DATA_VERSION_ID_0002 = 0xaf01 ; // current API
static const uint32_t RS_GXS_MSG_META_DATA_VERSION_ID_0002 = 0x0000 ; // current API
static const uint32_t RS_GXS_GRP_META_DATA_CURRENT_API_VERSION = RS_GXS_GRP_META_DATA_VERSION_ID_0002;
class RsGxsGrpMetaData
{
public:
@ -44,8 +51,8 @@ public:
RsGxsGrpMetaData();
bool deserialise(void *data, uint32_t &pktsize);
bool serialise(void* data, uint32_t &pktsize);
uint32_t serial_size();
bool serialise(void* data, uint32_t &pktsize, uint32_t api_version);
uint32_t serial_size(uint32_t api_version);
void clear();
void operator =(const RsGroupMetaData& rMeta);

2
libretroshare/src/gxs/rsgxsdataaccess.cc
View File

@ -1603,7 +1603,7 @@ bool RsGxsDataAccess::getServiceStatistic(ServiceStatisticRequest *req)
for(; mit != grpMeta.end(); ++mit)
{
RsGxsGrpMetaData* m = mit->second;
req->mServiceStatistic.mSizeOfGrps += m->mGrpSize + m->serial_size();
req->mServiceStatistic.mSizeOfGrps += m->mGrpSize + m->serial_size(RS_GXS_GRP_META_DATA_CURRENT_API_VERSION);
if (IS_GROUP_SUBSCRIBED(m->mSubscribeFlags))
{

1451
libretroshare/src/gxs/rsgxsnetservice.cc
View File

File diff suppressed because it is too large Load Diff

4
libretroshare/src/gxs/rsgxsnetservice.h
View File

@ -368,6 +368,7 @@ private:
void locked_doMsgUpdateWork(const RsNxsTransac* nxsTrans, const RsGxsGroupId& grpId);
void updateServerSyncTS();
void updateClientSyncTS();
bool locked_CanReceiveUpdate(const RsNxsSyncGrp* item);
bool locked_CanReceiveUpdate(const RsNxsSyncMsg* item);
@ -511,7 +512,8 @@ private:
RsGxsServerGrpUpdateItem* mGrpServerUpdateItem;
RsServiceInfo mServiceInfo;
void debugDump();
};
#endif // RSGXSNETSERVICE_H

288
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -27,8 +27,13 @@
#include "rsgxsutil.h"
#include "retroshare/rsgxsflags.h"
#include "retroshare/rspeers.h"
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
static const uint32_t MAX_GXS_IDS_REQUESTS_NET = 10 ; // max number of requests from cache/net (avoids killing the system!)
//#define GXSUTIL_DEBUG 1
RsGxsMessageCleanUp::RsGxsMessageCleanUp(RsGeneralDataService* const dataService, uint32_t messageStorePeriod, uint32_t chunkSize)
: mDs(dataService), MESSAGE_STORE_PERIOD(messageStorePeriod), CHUNK_SIZE(chunkSize)
@ -106,9 +111,8 @@ bool RsGxsMessageCleanUp::clean()
return mGrpMeta.empty();
}
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService) :
mDs(dataService), mDone(false), mIntegrityMutex("integrity")
RsGxsIntegrityCheck::RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs) :
mDs(dataService), mDone(false), mIntegrityMutex("integrity"),mGixs(gixs)
{ }
void RsGxsIntegrityCheck::run()
@ -118,122 +122,206 @@ void RsGxsIntegrityCheck::run()
bool RsGxsIntegrityCheck::check()
{
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
mDs->retrieveNxsGrps(grp, true, true);
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgIds;
GxsMsgReq grps;
// first take out all the groups
std::map<RsGxsGroupId, RsNxsGrp*> grp;
mDs->retrieveNxsGrps(grp, true, true);
std::vector<RsGxsGroupId> grpsToDel;
GxsMsgReq msgIds;
GxsMsgReq grps;
std::set<RsGxsId> used_gxs_ids ;
std::set<RsGxsGroupId> subscribed_groups ;
// compute hash and compare to stored value, if it fails then simply add it
// to list
std::map<RsGxsGroupId, RsNxsGrp*>::iterator git = grp.begin();
for(; git != grp.end(); ++git)
{
RsNxsGrp* grp = git->second;
RsFileHash currHash;
pqihash pHash;
pHash.addData(grp->grp.bin_data, grp->grp.bin_len);
pHash.Complete(currHash);
// compute hash and compare to stored value, if it fails then simply add it
// to list
std::map<RsGxsGroupId, RsNxsGrp*>::iterator git = grp.begin();
for(; git != grp.end(); ++git)
{
RsNxsGrp* grp = git->second;
RsFileHash currHash;
pqihash pHash;
pHash.addData(grp->grp.bin_data, grp->grp.bin_len);
pHash.Complete(currHash);
if(currHash == grp->metaData->mHash)
{
// get all message ids of group
if (mDs->retrieveMsgIds(grp->grpId, msgIds[grp->grpId]) == 1)
{
// store the group for retrieveNxsMsgs
grps[grp->grpId];
}
else
{
msgIds.erase(msgIds.find(grp->grpId));
// grpsToDel.push_back(grp->grpId);
}
}
else
{
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
if(currHash == grp->metaData->mHash)
{
// get all message ids of group
if (mDs->retrieveMsgIds(grp->grpId, msgIds[grp->grpId]) == 1)
{
// store the group for retrieveNxsMsgs
grps[grp->grpId];
mDs->removeGroups(grpsToDel);
if(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)
{
subscribed_groups.insert(git->first) ;
// now messages
GxsMsgReq msgsToDel;
GxsMsgResult msgs;
if(!grp->metaData->mAuthorId.isNull())
{
#ifdef GXSUTIL_DEBUG
std::cerr << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl;
#endif
mDs->retrieveNxsMsgs(grps, msgs, false, true);
used_gxs_ids.insert(grp->metaData->mAuthorId) ;
}
}
}
else
{
msgIds.erase(msgIds.find(grp->grpId));
// grpsToDel.push_back(grp->grpId);
}
// check msg ids and messages
GxsMsgReq::iterator msgIdsIt;
for (msgIdsIt = msgIds.begin(); msgIdsIt != msgIds.end(); ++msgIdsIt)
{
const RsGxsGroupId& grpId = msgIdsIt->first;
std::vector<RsGxsMessageId> &msgIdV = msgIdsIt->second;
}
else
{
grpsToDel.push_back(grp->grpId);
}
delete grp;
}
std::vector<RsGxsMessageId>::iterator msgIdIt;
for (msgIdIt = msgIdV.begin(); msgIdIt != msgIdV.end(); ++msgIdIt)
{
const RsGxsMessageId& msgId = *msgIdIt;
std::vector<RsNxsMsg*> &nxsMsgV = msgs[grpId];
mDs->removeGroups(grpsToDel);
std::vector<RsNxsMsg*>::iterator nxsMsgIt;
for (nxsMsgIt = nxsMsgV.begin(); nxsMsgIt != nxsMsgV.end(); ++nxsMsgIt)
{
RsNxsMsg *nxsMsg = *nxsMsgIt;
if (nxsMsg && msgId == nxsMsg->msgId)
{
break;
}
}
// now messages
GxsMsgReq msgsToDel;
GxsMsgResult msgs;
if (nxsMsgIt == nxsMsgV.end())
{
msgsToDel[grpId].push_back(msgId);
}
}
}
mDs->retrieveNxsMsgs(grps, msgs, false, true);
GxsMsgResult::iterator mit = msgs.begin();
// check msg ids and messages
GxsMsgReq::iterator msgIdsIt;
for (msgIdsIt = msgIds.begin(); msgIdsIt != msgIds.end(); ++msgIdsIt)
{
const RsGxsGroupId& grpId = msgIdsIt->first;
std::vector<RsGxsMessageId> &msgIdV = msgIdsIt->second;
for(; mit != msgs.end(); ++mit)
{
std::vector<RsNxsMsg*>& msgV = mit->second;
std::vector<RsNxsMsg*>::iterator vit = msgV.begin();
std::vector<RsGxsMessageId>::iterator msgIdIt;
for (msgIdIt = msgIdV.begin(); msgIdIt != msgIdV.end(); ++msgIdIt)
{
const RsGxsMessageId& msgId = *msgIdIt;
std::vector<RsNxsMsg*> &nxsMsgV = msgs[grpId];
for(; vit != msgV.end(); ++vit)
{
RsNxsMsg* msg = *vit;
RsFileHash currHash;
pqihash pHash;
pHash.addData(msg->msg.bin_data, msg->msg.bin_len);
pHash.Complete(currHash);
std::vector<RsNxsMsg*>::iterator nxsMsgIt;
for (nxsMsgIt = nxsMsgV.begin(); nxsMsgIt != nxsMsgV.end(); ++nxsMsgIt)
{
RsNxsMsg *nxsMsg = *nxsMsgIt;
if (nxsMsg && msgId == nxsMsg->msgId)
{
break;
}
}
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << "(EE) deleting message data with wrong hash or null meta data. meta=" << (void*)msg->metaData << std::endl;
msgsToDel[msg->grpId].push_back(msg->msgId);
}
if (nxsMsgIt == nxsMsgV.end())
{
msgsToDel[grpId].push_back(msgId);
}
}
}
delete msg;
}
}
GxsMsgResult::iterator mit = msgs.begin();
mDs->removeMsgs(msgsToDel);
for(; mit != msgs.end(); ++mit)
{
std::vector<RsNxsMsg*>& msgV = mit->second;
std::vector<RsNxsMsg*>::iterator vit = msgV.begin();
RsStackMutex stack(mIntegrityMutex);
mDone = true;
for(; vit != msgV.end(); ++vit)
{
RsNxsMsg* msg = *vit;
RsFileHash currHash;
pqihash pHash;
pHash.addData(msg->msg.bin_data, msg->msg.bin_len);
pHash.Complete(currHash);
std::vector<RsGxsGroupId>::iterator grpIt;
for(grpIt = grpsToDel.begin(); grpIt != grpsToDel.end(); ++grpIt)
{
mDeletedGrps.push_back(*grpIt);
}
mDeletedMsgs = msgsToDel;
if(msg->metaData == NULL || currHash != msg->metaData->mHash)
{
std::cerr << "(EE) deleting message data with wrong hash or null meta data. meta=" << (void*)msg->metaData << std::endl;
msgsToDel[msg->grpId].push_back(msg->msgId);
}
else if(!msg->metaData->mAuthorId.isNull() && subscribed_groups.find(msg->metaData->mGroupId)!=subscribed_groups.end())
{
#ifdef GXSUTIL_DEBUG
std::cerr << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl;
#endif
used_gxs_ids.insert(msg->metaData->mAuthorId) ;
}
return true;
delete msg;
}
}
mDs->removeMsgs(msgsToDel);
RsStackMutex stack(mIntegrityMutex);
mDone = true;
std::vector<RsGxsGroupId>::iterator grpIt;
for(grpIt = grpsToDel.begin(); grpIt != grpsToDel.end(); ++grpIt)
{
mDeletedGrps.push_back(*grpIt);
}
mDeletedMsgs = msgsToDel;
#ifdef GXSUTIL_DEBUG
std::cerr << "At end of pass, this is the list used GXS ids: " << std::endl;
std::cerr << " requesting them to GXS identity service to enforce loading." << std::endl;
#endif
std::list<RsPeerId> connected_friends ;
rsPeers->getOnlineList(connected_friends) ;
std::vector<RsGxsId> gxs_ids ;
for(std::set<RsGxsId>::const_iterator it(used_gxs_ids.begin());it!=used_gxs_ids.end();++it)
{
gxs_ids.push_back(*it) ;
#ifdef GXSUTIL_DEBUG
std::cerr << " " << *it << std::endl;
#endif
}
int nb_requested_not_in_cache = 0;
#ifdef GXSUTIL_DEBUG
std::cerr << " issuing random get on friends for non existing IDs" << std::endl;
#endif
// now request a cache update for them, which triggers downloading from friends, if missing.
for(;nb_requested_not_in_cache<MAX_GXS_IDS_REQUESTS_NET && gxs_ids.size()>0;)
{
uint32_t n = RSRandom::random_u32() % gxs_ids.size() ;
#ifdef GXSUTIL_DEBUG
std::cerr << " requesting ID " << gxs_ids[n] ;
#endif
if(!mGixs->haveKey(gxs_ids[n])) // checks if we have it already in the cache (conservative way to ensure that we atually have it)
{
mGixs->requestKey(gxs_ids[n],connected_friends);
++nb_requested_not_in_cache ;
#ifdef GXSUTIL_DEBUG
std::cerr << " ... from cache/net" << std::endl;
#endif
}
else
{
#ifdef GXSUTIL_DEBUG
std::cerr << " ... already in cache" << std::endl;
#endif
// Note: we could time_stamp even in the case where the id is not cached. Anyway, it's not really a problem here, since IDs have a high chance of
// behing eventually stamped.
mGixs->timeStampKey(gxs_ids[n]) ;
}
gxs_ids[n] = gxs_ids[gxs_ids.size()-1] ;
gxs_ids.pop_back() ;
}
#ifdef GXSUTIL_DEBUG
std::cerr << " total actual cache requests: "<< nb_requested_not_in_cache << std::endl;
#endif
return true;
}
bool RsGxsIntegrityCheck::isDone()

6
libretroshare/src/gxs/rsgxsutil.h
View File

@ -30,6 +30,8 @@
#include "serialiser/rsnxsitems.h"
#include "rsgds.h"
class RsGixs ;
/*!
* Handy function for cleaning out meta result containers
* @param container
@ -112,7 +114,7 @@ public:
* @param chunkSize
* @param sleepPeriod
*/
RsGxsIntegrityCheck(RsGeneralDataService* const dataService);
RsGxsIntegrityCheck(RsGeneralDataService* const dataService, RsGixs *gixs);
bool check();
@ -129,6 +131,8 @@ private:
RsMutex mIntegrityMutex;
std::list<RsGxsGroupId> mDeletedGrps;
std::map<RsGxsGroupId, std::vector<RsGxsMessageId> > mDeletedMsgs;
RsGixs *mGixs ;
};
class GroupUpdate

4
libretroshare/src/serialiser/rsgxsupdateitems.h
View File

@ -85,6 +85,8 @@ public:
struct MsgUpdateInfo
{
MsgUpdateInfo(): time_stamp(0), message_count(0) {}
uint32_t time_stamp ;
uint32_t message_count ;
};
@ -105,7 +107,7 @@ public:
virtual std::ostream &print(std::ostream &out, uint16_t indent);
RsGxsGroupId grpId;
uint32_t msgUpdateTS; // the last time this group received a new msg
uint32_t msgUpdateTS; // local time stamp this group last received a new msg
};