diff --git a/libretroshare/src/dbase/findex.cc b/libretroshare/src/dbase/findex.cc index ff080e3c8..f067f56af 100644 --- a/libretroshare/src/dbase/findex.cc +++ b/libretroshare/src/dbase/findex.cc @@ -26,6 +26,7 @@ #include "retroshare/rsexpr.h" #include "util/rsdir.h" #include "util/rsstring.h" +#include "util/rscompress.h" #include #include @@ -753,9 +754,10 @@ int FileIndex::printFileIndex(std::string &out) return 1; } -int FileIndex::loadIndex(const std::string& filename, const std::string& expectedHash, uint64_t /*size*/) +int FileIndex::loadIndex(const std::string& filename, const std::string& expectedHash, uint64_t size) { - std::ifstream file (filename.c_str(), std::ifstream::binary); + FILE *file = RsDirUtil::rs_fopen(filename.c_str(),"rb") ; + if (!file) { #ifdef FI_DEBUG @@ -766,28 +768,36 @@ int FileIndex::loadIndex(const std::string& filename, const std::string& expecte } /* load file into memory, close file */ - char ibuf[512]; - std::string s; - while(!file.eof()) - { - file.read(ibuf, 512); - s.append(ibuf, file.gcount()); - } - file.close(); + uint8_t *compressed_data = new uint8_t[size] ; - /* calculate hash */ - unsigned char sha_buf[SHA_DIGEST_LENGTH]; - SHA_CTX *sha_ctx = new SHA_CTX; - SHA1_Init(sha_ctx); - SHA1_Update(sha_ctx, s.c_str(), s.length()); - SHA1_Final(&sha_buf[0], sha_ctx); - delete sha_ctx; - - std::string tmpout; - for(int i = 0; i < SHA_DIGEST_LENGTH; i++) + if(compressed_data == NULL) { - rs_sprintf_append(tmpout, "%02x", (unsigned int) (sha_buf[i])); + std::cerr << "FileIndex::loadIndex(): can't allocate memory for " << size << " bytes." << std::endl; + return 0 ; } + int bytesread = 0 ; + if(size != (bytesread = fread(compressed_data,1,size,file))) + { + std::cerr << "FileIndex::loadIndex(): can't read " << size << " bytes from file " << filename << ". Only " << bytesread << " actually read." << std::endl; + return 0 ; + } + fclose(file) ; + + std::string tmpout = RsDirUtil::sha1sum((unsigned char *)(compressed_data),size).toStdString() ; + +// /* calculate hash */ +// unsigned char sha_buf[SHA_DIGEST_LENGTH]; +// SHA_CTX *sha_ctx = new SHA_CTX; +// SHA1_Init(sha_ctx); +// SHA1_Update(sha_ctx, s.c_str(), s.length()); +// SHA1_Final(&sha_buf[0], sha_ctx); +// delete sha_ctx; +// +// std::string tmpout; +// for(int i = 0; i < SHA_DIGEST_LENGTH; i++) +// { +// rs_sprintf_append(tmpout, "%02x", (unsigned int) (sha_buf[i])); +// } if (expectedHash != "" && expectedHash != tmpout) { @@ -798,6 +808,21 @@ int FileIndex::loadIndex(const std::string& filename, const std::string& expecte #endif return 0; } + // now uncompress the string + // + + uint8_t *uncompressed_data = NULL ; + unsigned int uncompressed_data_size = 0 ; + + if(!RsCompress::uncompress_memory_chunk(compressed_data,size,uncompressed_data,uncompressed_data_size)) + { + std::cerr << "FileIndex::loadIndex() Decompression failed! Fileindex can't be read." << std::endl; + return 0 ; + } + std::string s((char *)uncompressed_data,uncompressed_data_size) ; + + delete[] compressed_data ; + free(uncompressed_data) ; #define FIND_NEXT(s,start,end,c) end = s.find(c, start); if (end == std::string::npos) end = s.length(); @@ -973,7 +998,6 @@ int FileIndex::saveIndex(const std::string& filename, std::string &fileHash, uin std::string s; size = 0 ; - fileHash = "" ; /* print version and header */ s += "# FileIndex version 0.1\n"; @@ -1010,18 +1034,37 @@ int FileIndex::saveIndex(const std::string& filename, std::string &fileHash, uin /* signal to pop directory from stack in loadIndex() */ s += "-\n"; - /* calculate sha1 hash */ - SHA_CTX *sha_ctx = new SHA_CTX; - SHA1_Init(sha_ctx); - SHA1_Update(sha_ctx, s.c_str(), s.length()); - SHA1_Final(&sha_buf[0], sha_ctx); - delete sha_ctx; + // now compress the data. + + std::cerr << "FileIndex::saveIndex(): compressign data." << std::endl; - for(int i = 0; i < SHA_DIGEST_LENGTH; i++) + uint8_t *compressed_data = NULL ; + uint32_t compressed_data_size = 0 ; + + if(!RsCompress::compress_memory_chunk((unsigned char *)s.c_str(),s.length(),compressed_data,compressed_data_size)) { - rs_sprintf_append(fileHash, "%02x", (unsigned int) (sha_buf[i])); + std::cerr << "(EE) ERROR in file list compression ! file list can't be saved" << std::endl; + return false ; } + fileHash = RsDirUtil::sha1sum((unsigned char *)compressed_data,compressed_data_size).toStdString() ; + + std::cerr << " old size = " << s.length() << std::endl; + std::cerr << " new size = " << compressed_data_size << std::endl; + std::cerr << " hash = " << fileHash << std::endl; + +// /* calculate sha1 hash */ +// SHA_CTX *sha_ctx = new SHA_CTX; +// SHA1_Init(sha_ctx); +// SHA1_Update(sha_ctx, s.c_str(), s.length()); +// SHA1_Final(&sha_buf[0], sha_ctx); +// delete sha_ctx; +// +// for(int i = 0; i < SHA_DIGEST_LENGTH; i++) +// { +// rs_sprintf_append(fileHash, "%02x", (unsigned int) (sha_buf[i])); +// } + /* finally, save to file */ FILE *file = RsDirUtil::rs_fopen(filenametmp.c_str(), "wb"); @@ -1030,9 +1073,16 @@ int FileIndex::saveIndex(const std::string& filename, std::string &fileHash, uin std::cerr << "FileIndex::saveIndex error opening file for writting: " << filename << ". Giving up." << std::endl; return 0; } - fprintf(file,"%s",s.c_str()) ; + int outwritten ; + + if(compressed_data_size != (outwritten=fwrite(compressed_data,1,compressed_data_size,file))) + { + std::cerr << "FileIndex::saveIndex error. File not entirely written. Only " << outwritten << " bytes wrote out of " << compressed_data_size << " check for disk full, or disk quotas." << std::endl; + return 0; + } fclose(file); + free(compressed_data) ; // Use a temp file name so that the file is never half saved. // diff --git a/libretroshare/src/dht/p3bitdht.cc b/libretroshare/src/dht/p3bitdht.cc index 1bced9726..e366fbad3 100644 --- a/libretroshare/src/dht/p3bitdht.cc +++ b/libretroshare/src/dht/p3bitdht.cc @@ -349,7 +349,7 @@ bool p3BitDht::dropPeer(std::string pid) /* extract current peer status */ bool p3BitDht::getPeerStatus(std::string id, - struct sockaddr_in &/*laddr*/, struct sockaddr_in &/*raddr*/, + struct sockaddr_storage &/*laddr*/, struct sockaddr_storage &/*raddr*/, uint32_t &/*type*/, uint32_t &/*mode*/) { /* remove unused parameter warnings */ @@ -363,7 +363,7 @@ bool p3BitDht::getPeerStatus(std::string id, return false; } -bool p3BitDht::getExternalInterface(struct sockaddr_in &/*raddr*/, +bool p3BitDht::getExternalInterface(struct sockaddr_storage &/*raddr*/, uint32_t &/*mode*/) { diff --git a/libretroshare/src/dht/p3bitdht.h b/libretroshare/src/dht/p3bitdht.h index 608478c45..d53c9c745 100644 --- a/libretroshare/src/dht/p3bitdht.h +++ b/libretroshare/src/dht/p3bitdht.h @@ -197,8 +197,8 @@ virtual bool getNetworkStats(uint32_t &netsize, uint32_t &localnetsize); virtual bool findPeer(std::string id); virtual bool dropPeer(std::string id); -virtual int addBadPeer(const struct sockaddr_in &addr, uint32_t reason, uint32_t flags, uint32_t age); -virtual int addKnownPeer(const std::string &pid, const struct sockaddr_in &addr, uint32_t flags); +virtual int addBadPeer(const struct sockaddr_storage &addr, uint32_t reason, uint32_t flags, uint32_t age); +virtual int addKnownPeer(const std::string &pid, const struct sockaddr_storage &addr, uint32_t flags); //virtual int addFriend(const std::string pid); //virtual int addFriendOfFriend(const std::string pid); //virtual int addOther(const std::string pid); @@ -208,10 +208,10 @@ virtual void ConnectionFeedback(std::string pid, int state); /* extract current peer status */ virtual bool getPeerStatus(std::string id, - struct sockaddr_in &laddr, struct sockaddr_in &raddr, + struct sockaddr_storage &laddr, struct sockaddr_storage &raddr, uint32_t &type, uint32_t &mode); -virtual bool getExternalInterface(struct sockaddr_in &raddr, +virtual bool getExternalInterface(struct sockaddr_storage &raddr, uint32_t &mode); diff --git a/libretroshare/src/dht/p3bitdht_peernet.cc b/libretroshare/src/dht/p3bitdht_peernet.cc index 5420726f8..2a1915ddd 100644 --- a/libretroshare/src/dht/p3bitdht_peernet.cc +++ b/libretroshare/src/dht/p3bitdht_peernet.cc @@ -72,7 +72,13 @@ int p3BitDht::InfoCallback(const bdId *id, uint32_t /*type*/, uint32_t /*flags*/ if (mPeerSharer) { - mPeerSharer->updatePeer(rsid, addr, outtype, outreason, outage); + struct sockaddr_storage tmpaddr; + struct sockaddr_in *ap = (struct sockaddr_in *) &tmpaddr; + ap->sin_family = AF_INET; + ap->sin_addr = addr.sin_addr; + ap->sin_port = addr.sin_port; + + mPeerSharer->updatePeer(rsid, tmpaddr, outtype, outreason, outage); } /* call to the Stunners to drop the address as well */ @@ -1768,14 +1774,25 @@ int p3BitDht::checkConnectionAllowed(const bdId *peerId, int mode) */ -void p3BitDht::ConnectCalloutTCPAttempt(const std::string &peerId, struct sockaddr_in raddr) +void p3BitDht::ConnectCalloutTCPAttempt(const std::string &peerId, struct sockaddr_in raddrv4) { - struct sockaddr_in proxyaddr; - struct sockaddr_in srcaddr; - - sockaddr_clear(&proxyaddr); - sockaddr_clear(&srcaddr); + struct sockaddr_storage raddr; + struct sockaddr_storage proxyaddr; + struct sockaddr_storage srcaddr; + + sockaddr_storage_clear(proxyaddr); + sockaddr_storage_clear(proxyaddr); + sockaddr_storage_clear(srcaddr); + struct sockaddr_in *rap = (struct sockaddr_in *) &raddr; + struct sockaddr_in *pap = (struct sockaddr_in *) &proxyaddr; + struct sockaddr_in *sap = (struct sockaddr_in *) &srcaddr; + + // only one to translate + rap->sin_family = AF_INET; + rap->sin_addr = raddrv4.sin_addr; + rap->sin_port = raddrv4.sin_port; + uint32_t source = RS_CB_DHT; uint32_t connectFlags = RS_CB_FLAG_ORDER_UNSPEC | RS_CB_FLAG_MODE_TCP; uint32_t delay = 0; @@ -1785,14 +1802,25 @@ void p3BitDht::ConnectCalloutTCPAttempt(const std::string &peerId, struct sockad } -void p3BitDht::ConnectCalloutDirectOrProxy(const std::string &peerId, struct sockaddr_in raddr, uint32_t connectFlags, uint32_t delay) +void p3BitDht::ConnectCalloutDirectOrProxy(const std::string &peerId, struct sockaddr_in raddrv4, uint32_t connectFlags, uint32_t delay) { - struct sockaddr_in proxyaddr; - struct sockaddr_in srcaddr; - - sockaddr_clear(&proxyaddr); - sockaddr_clear(&srcaddr); - + struct sockaddr_storage raddr; + struct sockaddr_storage proxyaddr; + struct sockaddr_storage srcaddr; + + sockaddr_storage_clear(proxyaddr); + sockaddr_storage_clear(proxyaddr); + sockaddr_storage_clear(srcaddr); + + struct sockaddr_in *rap = (struct sockaddr_in *) &raddr; + struct sockaddr_in *pap = (struct sockaddr_in *) &proxyaddr; + struct sockaddr_in *sap = (struct sockaddr_in *) &srcaddr; + + // only one to translate + rap->sin_family = AF_INET; + rap->sin_addr = raddrv4.sin_addr; + rap->sin_port = raddrv4.sin_port; + uint32_t source = RS_CB_DHT; uint32_t bandwidth = 0; @@ -1800,9 +1828,33 @@ void p3BitDht::ConnectCalloutDirectOrProxy(const std::string &peerId, struct soc } void p3BitDht::ConnectCalloutRelay(const std::string &peerId, - struct sockaddr_in srcaddr, struct sockaddr_in proxyaddr, struct sockaddr_in destaddr, + struct sockaddr_in srcaddrv4, struct sockaddr_in proxyaddrv4, struct sockaddr_in destaddrv4, uint32_t connectFlags, uint32_t bandwidth) { + struct sockaddr_storage destaddr; + struct sockaddr_storage proxyaddr; + struct sockaddr_storage srcaddr; + + sockaddr_storage_clear(proxyaddr); + sockaddr_storage_clear(proxyaddr); + sockaddr_storage_clear(srcaddr); + + struct sockaddr_in *dap = (struct sockaddr_in *) &destaddr; + struct sockaddr_in *pap = (struct sockaddr_in *) &proxyaddr; + struct sockaddr_in *sap = (struct sockaddr_in *) &srcaddr; + + dap->sin_family = AF_INET; + dap->sin_addr = destaddrv4.sin_addr; + dap->sin_port = destaddrv4.sin_port; + + pap->sin_family = AF_INET; + pap->sin_addr = proxyaddrv4.sin_addr; + pap->sin_port = proxyaddrv4.sin_port; + + sap->sin_family = AF_INET; + sap->sin_addr = srcaddrv4.sin_addr; + sap->sin_port = srcaddrv4.sin_port; + uint32_t source = RS_CB_DHT; uint32_t delay = 0; diff --git a/libretroshare/src/dht/p3bitdht_peers.cc b/libretroshare/src/dht/p3bitdht_peers.cc index c16a742a9..1b5425209 100644 --- a/libretroshare/src/dht/p3bitdht_peers.cc +++ b/libretroshare/src/dht/p3bitdht_peers.cc @@ -184,26 +184,68 @@ bool p3BitDht::dropPeer(std::string pid) ********************************* Basic Peer Details ************************************* ******************************************************************************************/ -int p3BitDht::addBadPeer(const struct sockaddr_in &addr, uint32_t /*reason*/, uint32_t /*flags*/, uint32_t /*age*/) +int p3BitDht::addBadPeer(const struct sockaddr_storage &addr, uint32_t /*reason*/, uint32_t /*flags*/, uint32_t /*age*/) { //mUdpBitDht->updateKnownPeer(&id, 0, bdflags); + struct sockaddr_in addrv4; + if (addr.ss_family != AF_INET) + { + std::cerr << "p3BitDht::addBadPeer() cannot handle IPV6 Yet, aborting"; + std::cerr << std::endl; + abort(); + } + struct sockaddr_in *ap = (struct sockaddr_in *) &addr; + + // convert. + addrv4.sin_family = ap->sin_family; + addrv4.sin_addr = ap->sin_addr; + addrv4.sin_port = ap->sin_port; + + if (mDhtStunner) { - mDhtStunner->dropStunPeer(addr); + mDhtStunner->dropStunPeer(addrv4); } if (mProxyStunner) { - mProxyStunner->dropStunPeer(addr); + mProxyStunner->dropStunPeer(addrv4); } return 1; } -int p3BitDht::addKnownPeer(const std::string &pid, const struct sockaddr_in &addr, uint32_t flags) +int p3BitDht::addKnownPeer(const std::string &pid, const struct sockaddr_storage &addr, uint32_t flags) { + struct sockaddr_in addrv4; + if (addr.ss_family != AF_INET) + { + std::cerr << "p3BitDht::addKnownPeer() Warning! Non IPv4 Address - Cannot handle IPV6 Yet."; + std::cerr << std::endl; + sockaddr_clear(&addrv4); + + if (flags & NETASSIST_KNOWN_PEER_ONLINE) + { + std::cerr << "p3BitDht::addKnownPeer() Non IPv4 Address & ONLINE. Abort()ing."; + std::cerr << std::endl; + abort(); + } + } + else + { + + // convert. + struct sockaddr_in *ap = (struct sockaddr_in *) &addr; + + addrv4.sin_family = ap->sin_family; + addrv4.sin_addr = ap->sin_addr; + addrv4.sin_port = ap->sin_port; + } + + + int p3type = 0; int bdflags = 0; bdId id; @@ -262,13 +304,13 @@ int p3BitDht::addKnownPeer(const std::string &pid, const struct sockaddr_in &add id.id = dpd->mDhtId.id; - id.addr = addr; + id.addr = addrv4; } else { // shouldn't use own id without mutex - but it is static! id.id = mOwnDhtId; - id.addr = addr; + id.addr = addrv4; } mUdpBitDht->updateKnownPeer(&id, 0, bdflags); diff --git a/libretroshare/src/dht/stunaddrassist.h b/libretroshare/src/dht/stunaddrassist.h index 780267427..1df0bfabc 100644 --- a/libretroshare/src/dht/stunaddrassist.h +++ b/libretroshare/src/dht/stunaddrassist.h @@ -39,9 +39,21 @@ class stunAddrAssist: public pqiAddrAssist mStunner = stunner; } -virtual bool getExternalAddr(struct sockaddr_in &remote, uint8_t &stable) +virtual bool getExternalAddr(struct sockaddr_storage &remote, uint8_t &stable) { - return mStunner->externalAddr(remote, stable); + // IPV4 ONLY. + struct sockaddr_in remotev4; + if (mStunner->externalAddr(remotev4, stable)) + { + sockaddr_storage_clear(remote); + struct sockaddr_in *addr = (struct sockaddr_in *) &remote; + addr->sin_family = AF_INET; + addr->sin_addr = remotev4.sin_addr; + addr->sin_port = remotev4.sin_port; + + return true; + } + return false; } virtual int tick() diff --git a/libretroshare/src/distrib/p3distrib.cc b/libretroshare/src/distrib/p3distrib.cc deleted file mode 100644 index ca87b6af3..000000000 --- a/libretroshare/src/distrib/p3distrib.cc +++ /dev/null @@ -1,3802 +0,0 @@ -/* - * libretroshare/src/distrib: p3distrib.h - * - * - * Copyright 2004-2011 by Robert Fernie. - * 2010-2011 Christopher Evi-Parker - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifdef WINDOWS_SYS -#include "util/rswin.h" -#endif - -#include -#include - -#include -#include -#include -#include -#include -#include - -#include "rsserver/p3face.h" -#include "retroshare/rsdistrib.h" -#include "distrib/p3distrib.h" -#include "distrib/p3distribsecurity.h" -#include "serialiser/rsdistribitems.h" -#include "serialiser/rstlvkeys.h" - -#include "util/rsdir.h" -#include "pqi/pqibin.h" -#include "pqi/sslfns.h" -#include "pqi/authssl.h" -#include "pqi/authgpg.h" - -#include "retroshare/rspeers.h" // Needed for RsPeerDetails & Online List. (Should remove dependance) - -#define FAILED_CACHE_CONT "failedcachegrp" // cache id which have failed are stored under a node of this name/grpid -#define HIST_CACHE_FNAME "grp_history.xml" - -/*** - * #define ENABLE_CACHE_OPT 1 - ***/ - -/***** - * #define DISTRIB_DEBUG 1 - * #define DISTRIB_THREAD_DEBUG 1 - * #define DISTRIB_DUMMYMSG_DEBUG 1 - ****/ - -//#define DISTRIB_DEBUG 1 -//#define DISTRIB_THREAD_DEBUG 1 -//#define DISTRIB_DUMMYMSG_DEBUG 1 - - -GroupInfo::~GroupInfo() -{ - delete distribGroup ; - - for(std::map::const_iterator it(msgs.begin());it!=msgs.end();++it) - delete it->second ; - - for(std::map::const_iterator it(decrypted_msg_cache.begin());it!=decrypted_msg_cache.end();++it) - delete it->second ; -} - -p3GroupDistrib::p3GroupDistrib(uint16_t subtype, - CacheStrapper *cs, CacheTransfer *cft, - std::string sourcedir, std::string storedir, - std::string keyBackUpDir, uint32_t configId, - uint32_t storePeriod, uint32_t pubPeriod) - - :CacheSource(subtype, true, cs, sourcedir), - CacheStore(subtype, true, cs, cft, storedir), - p3Config(configId), p3ThreadedService(subtype), - mHistoricalCaches(true), distribMtx(""), - mStorePeriod(storePeriod), - mPubPeriod(pubPeriod), - mLastPublishTime(0), - mMaxCacheSubId(1), - mKeyBackUpDir(keyBackUpDir), BACKUP_KEY_FILE("key.log"), mLastKeyPublishTime(0), mLastRecvdKeyTime(0) -{ - /* force publication of groups (cleared if local cache file found) */ - mGroupsRepublish = true; - mGroupsChanged = true; - -#ifdef RSMUTEX_DEBUG - distribMtx.setName("p3GroupDistrib - " + keyBackUpDir.substr(keyBackUpDir.find_last_of('/') + 1)); -#endif - - mOwnId = AuthSSL::getAuthSSL()->OwnId(); - - addSerialType(new RsDistribSerialiser(getRsItemService(getType()))); - - return; -} - -p3GroupDistrib::~p3GroupDistrib() -{ - for(std::map::iterator it(mRecvdPubKeys.begin());it!=mRecvdPubKeys.end();++it) - delete it->second ; - - for(std::list::iterator it(mPendingPublish.begin());it!=mPendingPublish.end();++it) - delete *it ; -} - -int p3GroupDistrib::tick() -{ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::tick()"; - std::cerr << std::endl; -#endif - - time_t now = time(NULL); - bool toPublish; - - { - RsStackMutex stack(distribMtx); /**** STACK LOCKED MUTEX ****/ - - toPublish = ((mPendingPublish.size() > 0) || (mPendingPubKeyRecipients.size() > 0)) && (now > (time_t) (mPubPeriod + mLastPublishTime)); - } - - if (toPublish) - { - RsStackMutex stack(distribMtx); /**** STACK LOCKED MUTEX ****/ - - locked_publishPendingMsgs(); /* flags taken care of in here */ - } - - bool toPublishGroups; - { - RsStackMutex stack(distribMtx); /**** STACK LOCKED MUTEX ****/ - toPublishGroups = mGroupsRepublish; - } - - if (toPublishGroups) - { - publishDistribGroups(); - - IndicateConfigChanged(); /**** INDICATE CONFIG CHANGED! *****/ - - RsStackMutex stack(distribMtx); /**** STACK LOCKED MUTEX ****/ - mGroupsRepublish = false; - } - - bool attemptRecv = false; - - { - RsStackMutex stack(distribMtx); - toPublish = (mPendingPubKeyRecipients.size() > 0) && (now > (time_t) (mPubPeriod + mLastKeyPublishTime)); - attemptRecv = (mRecvdPubKeys.size() > 0); // attempt to load stored keys in case user has subscribed - } - - if(toPublish){ - RsStackMutex stack(distribMtx); - locked_sharePubKey(); - } - - - if(attemptRecv) - { - attemptPublishKeysRecvd(); - } - - bool toReceive = receivedItems(); - - if(toReceive){ - - receivePubKeys(); - } - - - return 0; -} - - -/***************************************************************************************/ -/***************************************************************************************/ - /********************** overloaded functions from Cache Store ******************/ -/***************************************************************************************/ -/***************************************************************************************/ - -int p3GroupDistrib::loadCache(const RsCacheData &data) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadCache()"; - std::cerr << std::endl; -#endif - - - RsStackMutex stack(distribMtx); - -#ifdef DISTRIB_THREAD_DEBUG - std::cerr << "p3GroupDistrib::loadCache() Storing PendingRemoteCache"; - std::cerr << std::endl; -#endif - - /* store the cache file for later processing */ - mPendingCaches.push_back(CacheDataPending(data, false, mHistoricalCaches)); - - if (data.size > 0) - { - CacheStore::lockData(); /***** LOCK ****/ - locked_storeCacheEntry(data); - CacheStore::unlockData(); /***** UNLOCK ****/ - } - - return 1; -} - -bool p3GroupDistrib::loadLocalCache(const RsCacheData &data) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadLocalCache()"; - std::cerr << std::endl; -#endif - - - RsStackMutex stack(distribMtx); - -#ifdef DISTRIB_THREAD_DEBUG - std::cerr << "p3GroupDistrib::loadCache() Storing PendingLocalCache"; - std::cerr << std::endl; -#endif - - /* store the cache file for later processing */ - mPendingCaches.push_back(CacheDataPending(data, true, mHistoricalCaches)); - - - if (data.size > 0) - { - refreshCache(data); - } - - return true; -} - - -/* Handle the Cache Pending Setup */ -CacheDataPending::CacheDataPending(const RsCacheData &data, bool local, bool historical) - :mData(data), mLocal(local), mHistorical(historical) -{ - return; -} - -bool p3GroupDistrib::isPeerAcceptedAsCacheProvider(const std::string& ssl_id) -{ - return rsPeers->servicePermissionFlags_sslid(ssl_id) & RS_SERVICE_PERM_DISTRIB ; -} - -bool p3GroupDistrib::isPeerAcceptedAsCacheReceiver(const std::string& ssl_id) -{ - return rsPeers->servicePermissionFlags_sslid(ssl_id) & RS_SERVICE_PERM_DISTRIB ; -} - -void p3GroupDistrib::HistoricalCachesDone() -{ - RsStackMutex stack(distribMtx); - std::string id; - mHistoricalCaches = false; // called when Stored Caches have been added to Pending List. -} - - /* From RsThread */ -void p3GroupDistrib::run() /* called once the thread is started */ -{ - -#ifdef DISTRIB_THREAD_DEBUG - std::cerr << "p3GroupDistrib::run()"; - std::cerr << std::endl; -#endif - -#ifdef DISTRIB_DUMMYMSG_DEBUG - int printed = 0; -#endif - RsCacheData cache; - while(isRunning()) - { - /* */ - - bool validCache = false; - bool isLocal = false; - bool isHistorical = false; - { - RsStackMutex stack(distribMtx); - - if (mPendingCaches.size() > 0) - { - CacheDataPending &pendingCache = mPendingCaches.front(); - cache = pendingCache.mData; - isLocal = pendingCache.mLocal; - isHistorical = pendingCache.mHistorical; - - validCache = true; - mPendingCaches.pop_front(); - -#ifdef DISTRIB_THREAD_DEBUG - std::cerr << "p3GroupDistrib::run() found pendingCache"; - std::cerr << std::endl; -#endif - - } - - } - if (validCache) - { - loadAnyCache(cache, isLocal, isHistorical); - -#ifndef WINDOWS_SYS - usleep(1000); -#else - Sleep(1); -#endif - } - else - { -#ifndef WINDOWS_SYS - sleep(1); -#else - Sleep(1000); -#endif - -#ifdef DISTRIB_DUMMYMSG_DEBUG - /* HACK for debugging */ - if (printed < 10) - { - RsStackMutex stack(distribMtx); - locked_printAllDummyMsgs(); - - printed++; - } -#endif - - } - } -} - - - -int p3GroupDistrib::loadAnyCache(const RsCacheData &data, bool local, bool historical) -{ - /* if subtype = 1 -> FileGroup, else -> FileMsgs */ - - std::string file = data.path; - file += "/"; - file += data.name; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadAnyCache() file: " << file << std::endl; - std::cerr << "PeerId: " << data.pid << std::endl; - std::cerr << "Cid: " << data.cid.type << ":" << data.cid.subid << std::endl; -#endif - - if (data.cid.subid == 1) - { - loadFileGroups(file, data.pid, local, historical); - } - else - { - loadFileMsgs(file, data, local, historical); - } - - return true; -} - -/***************************************************************************************/ -/***************************************************************************************/ - /********************** load Cache Files ***************************************/ -/***************************************************************************************/ -/***************************************************************************************/ - - -/* No need for special treatment for 'own' groups. - * configuration should be loaded before cache files. - */ -void p3GroupDistrib::loadFileGroups(const std::string &filename, const std::string &src, bool local, bool historical) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileGroups()"; - std::cerr << std::endl; -#endif - - /* create the serialiser to load info */ - BinInterface *bio = new BinFileInterface(filename.c_str(), BIN_FLAGS_READABLE); - pqistore *store = createStore(bio, src, BIN_FLAGS_READABLE); - -#ifdef DISTRIB_DEBUG - std::cerr << "loading file " << filename << std::endl ; -#endif - - RsItem *item; - RsDistribGrp *newGrp; - RsDistribGrpKey *newKey; - - while(NULL != (item = store->GetItem())) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileGroups() Got Item:"; - std::cerr << std::endl; - item->print(std::cerr, 10); - std::cerr << std::endl; -#endif - - newKey = dynamic_cast(item); - if ((newGrp = dynamic_cast(item))) - { - loadGroup(newGrp, historical); - } - else if ((newKey = dynamic_cast(item))) - { - loadGroupKey(newKey, historical); - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileGroups() Unexpected Item - deleting"; - std::cerr << std::endl; -#endif - delete item; - } - } - - delete store; - - if (local) - { - /* clear publication of groups if local cache file found */ - RsStackMutex stack(distribMtx); /******* STACK LOCKED MUTEX ***********/ - mGroupsRepublish = false; - } - - return; -} - -void p3GroupDistrib::loadFileMsgs(const std::string &filename, const RsCacheData& data, bool local, bool historical) -{ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs()"; - std::cerr << std::endl; -#endif - - uint16_t cacheSubId = data.cid.subid; - const std::string& src = data.pid; - uint32_t ts = data.recvd; - - time_t now = time(NULL); - - /* create the serialiser to load msgs */ - BinInterface *bio = new BinFileInterface(filename.c_str(), BIN_FLAGS_READABLE); - pqistore *store = createStore(bio, src, BIN_FLAGS_READABLE); - -#ifdef DISTRIB_DEBUG - std::cerr << "loading file " << filename << std::endl ; -#endif - - RsItem *item; - RsDistribSignedMsg *newMsg; - std::set grpAddedTo; - - while(isRunning() && (NULL != (item = store->GetItem()))) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs() Got Item:"; - std::cerr << std::endl; - item->print(std::cerr, 10); - std::cerr << std::endl; -#endif - - if ((newMsg = dynamic_cast(item))) - { - std::string& grpId = newMsg->grpId; - - bool ok; - { - RsStackMutex stack(distribMtx); - std::set::iterator it = mSubscribedGrp.find(grpId); - ok = it != mSubscribedGrp.end(); - } - - /* - * load msg if this is a subscribed group - * if not then add the grp cache map - */ - if(ok) - { - loadMsg(newMsg, src, local, historical); - }else - { - // add grp to set so cache not added to grp again - if(grpAddedTo.find(newMsg->grpId) == grpAddedTo.end()) - { - RsStackMutex stack(distribMtx); - mGrpCacheMap[grpId].push_back(data); - grpAddedTo.insert(grpId); - } - - delete item; - } - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs() Unexpected Item - deleting"; - std::cerr << std::endl; -#endif - /* wrong message type */ - delete item; - } - } - - - if (local) - { - /* now we create a map of time -> subid - * This is used to determine the newest and the oldest items - */ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs() Updating Local TimeStamps"; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::loadFileMsgs() CacheSubId: " << cacheSubId << " recvd: " << ts; - std::cerr << std::endl; -#endif - RsStackMutex stack(distribMtx); - - mLocalCacheTs[data.recvd] = cacheSubId; - if (cacheSubId > mMaxCacheSubId) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs() New Max CacheSubId"; - std::cerr << std::endl; -#endif - mMaxCacheSubId = cacheSubId; - } - - if (((time_t) ts < now) && ((time_t) ts > mLastPublishTime)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs() New LastPublishTime"; - std::cerr << std::endl; -#endif - mLastPublishTime = ts; - } - } - - delete store; - return; -} - - -bool p3GroupDistrib::processCacheOptReq(const std::string &grpId) -{ - { - RsStackMutex stack(distribMtx); - if(mSubscribedGrp.find(grpId) != mSubscribedGrp.end()) - return false; - - // grp already loaded - if(mCacheOptLoaded.find(grpId) != mCacheOptLoaded.end()) - return false; - } - - bool ok; - std::list cList; - - { - RsStackMutex stack(distribMtx); - CacheOptData::iterator mit = mGrpCacheMap.find(grpId); - ok = (mit != mGrpCacheMap.end()); - if(ok) cList = mit->second; - } - - if(ok) - { - std::list::iterator sit = cList.begin(); - - for(;sit != cList.end(); sit++) - loadCacheOptMsgs(*sit, grpId); - } - else - { - return false; - } - - RsStackMutex stack(distribMtx); - mCacheOptLoaded.insert(grpId); - - return true; -} - -void p3GroupDistrib::loadCacheOptMsgs(const RsCacheData& data, const std::string& grpId) -{ - std::string filename = data.path; - filename += "/"; - filename += data.name; - - /* create the serialiser to load msgs */ - BinInterface *bio = new BinFileInterface(filename.c_str(), BIN_FLAGS_READABLE); - pqistore *store = createStore(bio, data.pid, BIN_FLAGS_READABLE); - -#ifdef DISTRIB_DEBUG - std::cerr << "loading file " << filename << std::endl ; -#endif - - RsItem *item; - RsDistribSignedMsg *newMsg; - - while(isRunning() && (NULL != (item = store->GetItem()))) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs() Got Item:"; - std::cerr << std::endl; - item->print(std::cerr, 10); - std::cerr << std::endl; -#endif - - if ((newMsg = dynamic_cast(item))) - { - if(newMsg->grpId == grpId) - loadMsg(newMsg, data.pid, false, true); - else - delete item; - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadFileMsgs() Unexpected Item - deleting"; - std::cerr << std::endl; -#endif - /* wrong message type */ - delete item; - } - } - - delete store; - return; -} - - -/***************************************************************************************/ -/***************************************************************************************/ - /********************** load Cache Msgs ***************************************/ -/***************************************************************************************/ -/***************************************************************************************/ - - -bool p3GroupDistrib::loadGroup(RsDistribGrp *newGrp, bool historical) -{ - /* load groupInfo */ - const std::string &gid = newGrp -> grpId; - const std::string &pid = newGrp -> PeerId(); - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroup()" << std::endl; - std::cerr << "groupId: " << gid << std::endl; - std::cerr << "PeerId: " << pid << std::endl; - std::cerr << "Group:" << std::endl; - newGrp -> print(std::cerr, 10); - std::cerr << "----------------------" << std::endl; -#endif - - RsStackMutex stack(distribMtx); /******* STACK LOCKED MUTEX ***********/ - - /* look for duplicate */ - bool checked = false; - bool isNew = false; - bool ok = false; - std::map::iterator it; - it = mGroups.find(gid); - - if (it == mGroups.end()) - { - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroup() Group Not Found"; - std::cerr << std::endl; -#endif - - if (!p3DistribSecurity::validateDistribGrp(newGrp)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroup() Invalid Group "; - std::cerr << std::endl; -#endif - /* fails test */ - delete newGrp; - return false; - } - - checked = true; - - GroupInfo gi; - gi.grpId = gid; - mGroups[gid] = gi; - - it = mGroups.find(gid); - - isNew = true; - } - - /* at this point - always in the map */ - - /* add as source ... don't need to validate for this! */ - std::list::iterator pit; - pit = std::find(it->second.sources.begin(), it->second.sources.end(), pid); - if (pit == it->second.sources.end()) - { - it->second.sources.push_back(pid); - it->second.pop = it->second.sources.size(); - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroup() New Source, pop = "; - std::cerr << it->second.pop; - std::cerr << std::endl; -#endif - } - - if (!checked) - { - if (!locked_checkGroupInfo(it->second, newGrp)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroup() Fails Check"; - std::cerr << std::endl; -#endif - /* either fails check or old/same data */ - delete newGrp; - return false; - } - } - - /* useful info/update */ - if(!locked_updateGroupInfo(it->second, newGrp)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroup() Fails Update"; - std::cerr << std::endl; -#endif - /* cleanup on false */ - delete newGrp; - } - else - { - /* Callback for any derived classes */ - - if (isNew) - locked_notifyGroupChanged(it->second, GRP_NEW_UPDATE, historical); - else - locked_notifyGroupChanged(it->second, GRP_UPDATE, historical); - - ok = true; - } - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroup() Done"; - std::cerr << std::endl; - std::cerr << "ok: " << ok << std::endl; -#endif - return ok; - -} - - -bool p3GroupDistrib::loadGroupKey(RsDistribGrpKey *newKey, bool historical) -{ - /* load Key */ - const std::string &gid = newKey -> grpId; - -#ifdef DISTRIB_DEBUG - const std::string &pid = newKey -> PeerId(); - std::cerr << "p3GroupDistrib::loadGroupKey()" << std::endl; - std::cerr << "PeerId: " << pid << std::endl; - std::cerr << "groupId: " << gid << std::endl; - std::cerr << "Key:" << std::endl; - newKey -> print(std::cerr, 10); - std::cerr << "----------------------" << std::endl; -#endif - - RsStackMutex stack(distribMtx); /******* STACK LOCKED MUTEX ***********/ - - /* Find the Group */ - std::map::iterator it; - it = mGroups.find(gid); - - // - if (it == mGroups.end()) - { - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroupKey() Group for key not found: discarding"; - std::cerr << std::endl; -#endif - - delete newKey; - newKey = NULL; - return false; - } - - /* have the group -> add in the key */ - bool updateOk = false; - if (newKey->key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) - { - if(!locked_updateGroupAdminKey(it->second, newKey)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroupKey() Failed Admin Key Update"; - std::cerr << std::endl; -#endif - } - else - { - updateOk = true; - } - } - else - { - if(!locked_updateGroupPublishKey(it->second, newKey)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroupKey() Failed Publish Key Update"; - std::cerr << std::endl; -#endif - } - else - { - updateOk = true; - } - } - - if (updateOk) - locked_notifyGroupChanged(it->second, GRP_LOAD_KEY, historical); - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadGroupKey() Done - Cleaning up."; - std::cerr << std::endl; -#endif - -// if(!updateOk) - delete newKey; - - newKey = NULL; - return updateOk; -} - - -bool p3GroupDistrib::loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, bool local, bool historical) -{ - /****************** check the msg ******************/ - /* Do the most likely checks to fail first.... - * - * timestamp (too old) - * group (non existant) - * msg (already have it) - * - * -> then do the expensive Hash / signature checks. - */ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg()" << std::endl; - std::cerr << "Source:" << src << std::endl; - std::cerr << "Local:" << local << std::endl; - newMsg -> print(std::cerr, 10); - std::cerr << "----------------------" << std::endl; -#endif - - RsStackMutex stack(distribMtx); /******* STACK LOCKED MUTEX ***********/ - - /* Check if it exists already */ - - /* find group */ - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(newMsg->grpId))) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() Group Dont Exist" << std::endl; - std::cerr << std::endl; -#endif - /* if not there -> remove */ - delete newMsg; - return false; - } - - /****************** check the msg ******************/ - /* check for duplicate message, do this first to ensure minimal signature validations. - * therefore, duplicateMsg... could potentially be called on a dodgey msg (not a big problem!) - */ - - std::map::iterator mit; - mit = (git->second).msgs.find(newMsg->msgId); - if (mit != (git->second).msgs.end()) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() Msg already exists" << std::endl; - std::cerr << std::endl; -#endif - /* if already there -> remove */ - locked_eventDuplicateMsg(&(git->second), mit->second, src, historical); - delete newMsg; - return false; - } - - /* if unique (new) msg - do validation */ - if (!p3DistribSecurity::validateDistribSignedMsg(git->second, newMsg)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() validate failed" << std::endl; - std::cerr << std::endl; -#endif - delete newMsg; - return false; - } - - void *temp_ptr = newMsg->packet.bin_data; - int temp_len = newMsg->packet.bin_len; - - /* if encrypted data then decrypt */ - if(git->second.grpFlags & RS_DISTRIB_ENCRYPTED){ - void *out_data = NULL; - int out_len = 0; - - EVP_PKEY * privateKey = NULL; - std::map::iterator kit; - - for(kit = git->second.publishKeys.begin(); kit != git->second - .publishKeys.end(); kit++ ){ - // Does not allow for possibility of different keys - - if((kit->second.type & RSTLV_KEY_TYPE_FULL) && (kit->second.key->type == EVP_PKEY_RSA)){ - privateKey = kit->second.key; - break; - } - } - - if(kit == git->second.publishKeys.end()){ - #ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg(): Cannot find full key, grpId " << grpId - << std::endl; - #endif - delete newMsg; - return false; - } - - - if(p3DistribSecurity::decrypt(out_data, out_len, newMsg->packet.bin_data, newMsg->packet.bin_len, privateKey)){ - - newMsg->packet.TlvShallowClear(); - newMsg->packet.setBinData(out_data, out_len); - delete[] (unsigned char*) out_data; - - }else{ - if((out_data != NULL) && (out_len != 0)) - delete[] (unsigned char*) out_data; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() Failed to decrypt" << std::endl; - std::cerr << std::endl; -#endif - delete newMsg; - return false; - } - } - - /* convert Msg */ - RsDistribMsg *msg = unpackDistribSignedMsg(newMsg); - - if (!msg) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() unpack failed" << std::endl; - std::cerr << std::endl; -#endif - delete newMsg; - return false; - } - - if (!locked_checkDistribMsg(git->second, msg)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() check failed" << std::endl; - std::cerr << std::endl; -#endif - delete newMsg; - delete msg; - return false; - } - - /* accept message */ - (git->second).msgs[msg->msgId] = msg; - - // update the time stamp of group for last post - if((git->second.lastPost < (time_t)msg->timestamp)) - git->second.lastPost = msg->timestamp; - - // Interface to handle Dummy Msgs. - locked_CheckNewMsgDummies(git->second, msg, src, historical); - - /* now update parents TS */ - locked_updateChildTS(git->second, msg); - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() Msg Loaded Successfully" << std::endl; - std::cerr << std::endl; -#endif - - /* Callback for any derived classes to play with */ - locked_eventNewMsg(&(git->second), msg, src, historical); - - /* else if group = subscribed | listener -> publish */ - /* if it has come from us... then it has been published already */ - if ((!local) && (git->second.flags & (RS_DISTRIB_SUBSCRIBED))) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() To be Published!"; - std::cerr << std::endl; -#endif - - if(git->second.grpFlags & RS_DISTRIB_ENCRYPTED){ - newMsg->packet.TlvClear(); - newMsg->packet.setBinData(temp_ptr, temp_len); - } - - locked_toPublishMsg(newMsg); - } - else - { - /* Note it makes it very difficult to republish msg - if we have - * deleted the signed version... The load of old messages will occur - * at next startup. And publication will happen then too. - */ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::loadMsg() Deleted Original Msg (No Publish)"; - std::cerr << std::endl; -#endif - delete newMsg; - - } - - if(!historical) - locked_notifyGroupChanged(git->second, GRP_NEW_MSG, historical); - - return true; -} - -/***************************************************************************************/ -/***************************************************************************************/ - /****************** create/mod Cache Content **********************************/ -/***************************************************************************************/ -/***************************************************************************************/ - -void p3GroupDistrib::locked_toPublishMsg(RsDistribSignedMsg *msg) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_toPublishMsg() Adding to PendingPublish List"; - std::cerr << std::endl; -#endif - mPendingPublish.push_back(msg); - if (msg->PeerId() == mOwnId) - { - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_toPublishMsg() Local -> ConfigSave Requested"; - std::cerr << std::endl; -#endif - /* we need to trigger Configuration save */ - IndicateConfigChanged(); /**** INDICATE CONFIG CHANGED! *****/ - } -} - - -uint16_t p3GroupDistrib::locked_determineCacheSubId() -{ - /* if oldest cache is previous to StorePeriod - use that */ - time_t now = time(NULL); - uint16_t id = 1; - - uint32_t oldest = now; - if (mLocalCacheTs.size() > 0) - { - oldest = mLocalCacheTs.begin()->first; - } - - if (oldest < now - mStorePeriod) - { - /* clear it out, return id */ - id = mLocalCacheTs.begin()->second; - mLocalCacheTs.erase(mLocalCacheTs.begin()); - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_determineCacheSubId() Replacing Old CacheId: " << id; - std::cerr << std::endl; -#endif - return id; - } - - mMaxCacheSubId++; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_determineCacheSubId() Returning new Id: " << mMaxCacheSubId; - std::cerr << std::endl; -#endif - /* else return maximum */ - return mMaxCacheSubId; -} - - -void p3GroupDistrib::locked_publishPendingMsgs() -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_publishPendingMsgs()"; - std::cerr << std::endl; -#endif - /* get the next message id */ - RsCacheData newCache; - time_t now = time(NULL); - - bool ok = true; // hass msg/cache file been written successfully - - newCache.pid = mOwnId; - newCache.cid.type = CacheSource::getCacheType(); - newCache.cid.subid = locked_determineCacheSubId(); - - /* create filename */ - std::string path = CacheSource::getCacheDir(); - - std::string tmpname; - rs_sprintf(tmpname, "grpdist-t%u-msgs-%ld.dist", CacheSource::getCacheType(), time(NULL)); - std::string filename = path + "/" + tmpname ; - std::string filenametmp = path + "/" + tmpname + ".tmp"; - - BinInterface *bio = new BinFileInterface(filenametmp.c_str(), BIN_FLAGS_WRITEABLE | BIN_FLAGS_HASH_DATA); - pqistore *store = createStore(bio, mOwnId, BIN_FLAGS_WRITEABLE); /* messages are deleted! */ - - bool resave = false; - std::list::iterator it; - for(it = mPendingPublish.begin(); it != mPendingPublish.end(); it++) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_publishPendingMsgs() Publishing:"; - std::cerr << std::endl; - (*it)->print(std::cerr, 10); - std::cerr << std::endl; -#endif - if ((*it)->PeerId() == mOwnId) - { - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_publishPendingMsgs() Own Publish"; - std::cerr << std::endl; -#endif - resave = true; - } - - // prevent sending original source of message to peers - (*it)->PeerId(mOwnId); - - if(!store->SendItem(*it)) /* deletes it */ - { - ok &= false; - } - } - - /* Extract File Information from pqistore */ - newCache.path = path; - newCache.name = tmpname; - - newCache.hash = bio->gethash(); - newCache.size = bio->bytecount(); - newCache.recvd = now; - - /* cleanup */ - mPendingPublish.clear(); - delete store; - - if(!RsDirUtil::renameFile(filenametmp,filename)) - { - std::string errlog; - ok &= false; -#ifdef WIN32 - rs_sprintf(errlog, "Error %u", GetLastError()); -#else - rs_sprintf(errlog, "Error %d", errno); -#endif - RsServer::notify()->AddSysMessage(0, RS_SYS_WARNING, "File rename error", "Error while renaming file " + filename + ": got error " + errlog); - } - - /* indicate not to save for a while */ - mLastPublishTime = now; - - /* push file to CacheSource */ - - if(ok) - refreshCache(newCache); - - - if (ok && resave) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_publishPendingMsgs() Indicate Save Data Changed"; - std::cerr << std::endl; -#endif - /* flag to store config (saying we've published messages) */ - IndicateConfigChanged(); /**** INDICATE CONFIG CHANGED! *****/ - } -} - - -void p3GroupDistrib::publishDistribGroups() -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishDistribGroups()"; - std::cerr << std::endl; -#endif - - /* set subid = 1 */ - RsCacheData newCache; - - newCache.pid = mOwnId; - newCache.cid.type = CacheSource::getCacheType(); - newCache.cid.subid = 1; - - /* create filename */ - std::string path = CacheSource::getCacheDir(); - - std::string tmpname; - rs_sprintf(tmpname, "grpdist-t%u-grps-%ld.dist", CacheSource::getCacheType(), time(NULL)); - std::string filename = path + "/" + tmpname; - std::string filenametmp = path + "/" + tmpname + ".tmp"; - std::string tempPeerId; // to store actual id temporarily - - BinInterface *bio = new BinFileInterface(filenametmp.c_str(), BIN_FLAGS_WRITEABLE | BIN_FLAGS_HASH_DATA); - pqistore *store = createStore(bio, mOwnId, BIN_FLAGS_NO_DELETE | BIN_FLAGS_WRITEABLE); - - RsStackMutex stack(distribMtx); /****** STACK MUTEX LOCKED *******/ - - - /* Iterate through all the Groups */ - std::map::iterator it; - for(it = mGroups.begin(); it != mGroups.end(); it++) - { - /* if subscribed or listener or admin -> then send it to be published by cache */ - if ((it->second.flags & RS_DISTRIB_SUBSCRIBED) || (it->second.flags & RS_DISTRIB_ADMIN)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishDistribGroups() Saving Group: " << it->first; - std::cerr << std::endl; -#endif - - /* extract public info to RsDistribGrp */ - RsDistribGrp *grp = it->second.distribGroup; - - if (grp) - { - /* store in Cache File */ - tempPeerId = grp->PeerId(); - grp->PeerId(mOwnId); // prevent sending original source to users - store->SendItem(grp); /* no delete */ - grp->PeerId(tempPeerId); - - } - - /* if they have public keys, publish these too */ - std::map::iterator kit; - for(kit = it->second.publishKeys.begin(); - kit != it->second.publishKeys.end(); kit++) - { - if ((kit->second.type & RSTLV_KEY_DISTRIB_PUBLIC) && - (kit->second.type & RSTLV_KEY_TYPE_FULL)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishDistribGroups() Saving Key: " << kit->first; - std::cerr << std::endl; -#endif - /* create Key for sharing */ - - RsDistribGrpKey *pubKey = new RsDistribGrpKey(); - pubKey->grpId = it->first; - - RSA *rsa_priv = EVP_PKEY_get1_RSA(kit->second.key); - p3DistribSecurity::setRSAPrivateKey(pubKey->key, rsa_priv); - RSA_free(rsa_priv); - - pubKey->key.keyFlags = RSTLV_KEY_TYPE_FULL; - pubKey->key.keyFlags |= RSTLV_KEY_DISTRIB_PUBLIC; - pubKey->key.startTS = kit->second.startTS; - pubKey->key.endTS = kit->second.endTS; - - store->SendItem(pubKey); - delete pubKey; - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishDistribGroups() Ignoring Key: " << kit->first; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::publishDistribGroups() Key Type: " << kit->second.type; - std::cerr << std::endl; -#endif - } - - } - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishDistribGroups() Ignoring Group: " << it->first; - std::cerr << std::endl; -#endif - } - - } - - /* Extract File Information from pqistore */ - newCache.path = path; - newCache.name = tmpname; - - newCache.hash = bio->gethash(); - newCache.size = bio->bytecount(); - newCache.recvd = time(NULL); - - /* cleanup */ - delete store; - - if(!RsDirUtil::renameFile(filenametmp,filename)) - { - std::string errlog; -#ifdef WIN32 - rs_sprintf(errlog, "Error %u", GetLastError()); -#else - rs_sprintf(errlog, "Error %d", errno); -#endif - RsServer::notify()->AddSysMessage(0, RS_SYS_WARNING, "File rename error", "Error while renaming file " + filename + ": got error " + errlog); - } - - /* push file to CacheSource */ - refreshCache(newCache); -} - - - /* clearing old data */ -void p3GroupDistrib::clear_local_caches(time_t now) -{ - RsStackMutex stack(distribMtx); /****** STACK MUTEX LOCKED *******/ - - time_t cutoff = now - mStorePeriod; - std::list::iterator it; - for(it = mLocalCaches.begin(); it != mLocalCaches.end();) - { - if (it->end < cutoff) - { - /* Call to CacheSource Function */ - CacheId cid(CacheSource::getCacheType(), it->cacheSubId); - clearCache(cid); - it = mLocalCaches.erase(it); - } - else - { - it++; - } - } -} - - - -/***************************************************************************************/ -/***************************************************************************************/ - /********************** Access Content ***************************************/ -/***************************************************************************************/ -/***************************************************************************************/ - - -/* get Group Lists */ -bool p3GroupDistrib::getAllGroupList(std::list &grpids) -{ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - std::map::iterator git; - for(git = mGroups.begin(); git != mGroups.end(); git++) - { - grpids.push_back(git->first); - } - return true; -} - -bool p3GroupDistrib::getSubscribedGroupList(std::list &grpids) -{ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - std::map::iterator git; - for(git = mGroups.begin(); git != mGroups.end(); git++) - { - if (git->second.flags & RS_DISTRIB_SUBSCRIBED) - { - grpids.push_back(git->first); - } - } - return true; -} - -bool p3GroupDistrib::getPublishGroupList(std::list &grpids) -{ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - std::map::iterator git; - for(git = mGroups.begin(); git != mGroups.end(); git++) - { - if (git->second.flags & (RS_DISTRIB_ADMIN | RS_DISTRIB_PUBLISH)) - { - grpids.push_back(git->first); - } - } - return true; -} - -void p3GroupDistrib::getPopularGroupList(uint32_t popMin, uint32_t popMax, std::list &grpids) -{ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - std::map::iterator git; - for(git = mGroups.begin(); git != mGroups.end(); git++) - { - if ((git->second.pop >= popMin) && - (git->second.pop <= popMax)) - { - grpids.push_back(git->first); - } - } - return; -} - - -/* get Msg Lists */ -bool p3GroupDistrib::getAllMsgList(const std::string& grpId, std::list &msgIds) -{ - - processCacheOptReq(grpId); - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - - - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { - return false; - } - - std::map::iterator mit; - std::map msgs; - - for(mit = git->second.msgs.begin(); mit != git->second.msgs.end(); mit++) - { - msgIds.push_back(mit->first); - } - return true; -} - -bool p3GroupDistrib::getParentMsgList(const std::string& grpId, const std::string& pId, - std::list &msgIds) -{ - processCacheOptReq(grpId); - - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { - return false; - } - - std::map::iterator mit; - - for(mit = git->second.msgs.begin(); mit != git->second.msgs.end(); mit++) - { - if (mit->second->parentId == pId) - { - msgIds.push_back(mit->first); - } - } - return true; -} - -bool p3GroupDistrib::getTimePeriodMsgList(const std::string& grpId, uint32_t timeMin, - uint32_t timeMax, std::list &msgIds) -{ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { - return false; - } - - std::map::iterator mit; - - for(mit = git->second.msgs.begin(); mit != git->second.msgs.end(); mit++) - { - if ((mit->second->timestamp >= timeMin) && - (mit->second->timestamp <= timeMax)) - { - msgIds.push_back(mit->first); - } - } - return true; -} - - -GroupInfo *p3GroupDistrib::locked_getGroupInfo(const std::string& grpId) -{ - /************* ALREADY LOCKED ************/ - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { - return NULL; - } - return &(git->second); -} - - -RsDistribMsg *p3GroupDistrib::locked_getGroupMsg(const std::string& grpId, const std::string& msgId) -{ - - /************* ALREADY LOCKED ************/ - - - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { - return NULL; - } - - std::map::iterator mit; - if (git->second.msgs.end() == (mit = git->second.msgs.find(msgId))) - { - return NULL; - } - - return mit->second; -} - -bool p3GroupDistrib::subscribeToGroup(const std::string &grpId, bool subscribe) -{ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { - return false; - } - - if (subscribe) - { - if (!(git->second.flags & RS_DISTRIB_SUBSCRIBED)) - { - git->second.flags |= RS_DISTRIB_SUBSCRIBED; - - locked_notifyGroupChanged(git->second, GRP_SUBSCRIBED, false); - mGroupsRepublish = true; - - /* reprocess groups messages .... so actions can be taken (by inherited) - * This could be an very expensive operation! .... but they asked for it. - * - * Hopefully a LoadList call will have on existing messages! - */ - - std::map::iterator mit; - std::list::iterator pit; - - /* assume that each peer can provide all of them */ - for(mit = git->second.msgs.begin(); - mit != git->second.msgs.end(); mit++) - { - for(pit = git->second.sources.begin(); - pit != git->second.sources.end(); pit++) - { - if(*pit != mOwnId) - locked_eventDuplicateMsg(&(git->second), mit->second, *pit, false); - } - } - } - } - else - { - if (git->second.flags & RS_DISTRIB_SUBSCRIBED) - { - git->second.flags &= (~RS_DISTRIB_SUBSCRIBED); - - locked_notifyGroupChanged(git->second, GRP_UNSUBSCRIBED, false); - mGroupsRepublish = true; - } - } - - return true; -} - - -bool p3GroupDistrib::attemptPublishKeysRecvd() -{ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::attemptPublishKeysRecvd() " << std::endl; -#endif - - RsStackMutex stack(distribMtx); - - std::list toDelete; - std::list::iterator sit; - std::map::iterator it; - - std::map::iterator mit; - mit = mRecvdPubKeys.begin(); - - // add received keys for groups that are present - for(; mit != mRecvdPubKeys.end(); mit++){ - - it = mGroups.find(mit->first); - - // group has not arrived yet don't attempt to add - if(it == mGroups.end()){ - - continue; - - }else{ - - - - if(locked_updateGroupPublishKey(it->second, mit->second)){ - - locked_notifyGroupChanged(it->second, GRP_LOAD_KEY, false); - } - - if(it->second.flags & RS_DISTRIB_SUBSCRIBED){ - // remove key shared flag so key is not loaded back into mrecvdpubkeys - mit->second->key.keyFlags &= (~RSTLV_KEY_TYPE_SHARED); - - delete (mit->second); - toDelete.push_back(mit->first); - } - - } - } - - sit = toDelete.begin(); - - for(; sit != toDelete.end(); sit++) - mRecvdPubKeys.erase(*sit); - - if(!toDelete.empty()) IndicateConfigChanged(); - - - return true; -} - -/************************************* p3Config *************************************/ - -RsSerialiser *p3GroupDistrib::setupSerialiser() -{ - RsSerialiser *rss = new RsSerialiser(); - rss->addSerialType(new RsDistribSerialiser()); - - return rss; -} - -bool p3GroupDistrib::saveList(bool &cleanup, std::list& saveData) -{ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::saveList()"; - std::cerr << std::endl; -#endif - - cleanup = false; - - distribMtx.lock(); /****** MUTEX LOCKED *******/ - - /* Iterate through all the Groups */ - std::map::iterator it; - for(it = mGroups.begin(); it != mGroups.end(); it++) - { - /* if subscribed or listener -> do stuff */ - if (it->second.flags & (RS_DISTRIB_SUBSCRIBED)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::saveList() Saving Group: " << it->first; - std::cerr << std::endl; -#endif - - /* extract public info to RsDistribGrp */ - RsDistribGrp *grp = it->second.distribGroup; - - if (grp) - { - /* store in Cache File */ - saveData.push_back(grp); /* no delete */ - } - - /* if they have public keys, publish these too */ - std::map::iterator kit; - for(kit = it->second.publishKeys.begin(); - kit != it->second.publishKeys.end(); kit++) - { - if (kit->second.type & RSTLV_KEY_TYPE_FULL) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::saveList() Saving Key: " << kit->first; - std::cerr << std::endl; -#endif - /* create Key for sharing */ - - RsDistribGrpKey *pubKey = new RsDistribGrpKey(); - pubKey->grpId = it->first; - - RSA *rsa_priv = EVP_PKEY_get1_RSA(kit->second.key); - p3DistribSecurity::setRSAPrivateKey(pubKey->key, rsa_priv); - RSA_free(rsa_priv); - - pubKey->key.keyFlags = kit->second.type; - pubKey->key.startTS = kit->second.startTS; - pubKey->key.endTS = kit->second.endTS; - - saveData.push_back(pubKey); - saveCleanupList.push_back(pubKey); - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::saveList() Ignoring Key: " << kit->first; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::saveList() Key Type: " << kit->second.type; - std::cerr << std::endl; -#endif - } - - } - - if (it->second.adminKey.type & RSTLV_KEY_TYPE_FULL) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::saveList() Saving Admin Key"; - std::cerr << std::endl; -#endif - /* create Key for sharing */ - - RsDistribGrpKey *pubKey = new RsDistribGrpKey(); - pubKey->grpId = it->first; - - RSA *rsa_priv = EVP_PKEY_get1_RSA(kit->second.key); - p3DistribSecurity::setRSAPrivateKey(pubKey->key, rsa_priv); - RSA_free(rsa_priv); - - pubKey->key.keyFlags = RSTLV_KEY_TYPE_FULL; - pubKey->key.keyFlags |= RSTLV_KEY_DISTRIB_ADMIN; - pubKey->key.startTS = kit->second.startTS; - pubKey->key.endTS = kit->second.endTS; - - saveData.push_back(pubKey); - saveCleanupList.push_back(pubKey); - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::saveList() Ignoring Admin Key: " << it->second.adminKey.keyId; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::saveList() Admin Key Type: " << it->second.adminKey.type; - std::cerr << std::endl; -#endif - } - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::saveList() Ignoring Group: " << it->first; - std::cerr << std::endl; -#endif - } - - } - - std::list::iterator mit; - for(mit = mPendingPublish.begin(); mit != mPendingPublish.end(); mit++) - { - if ((*mit)->PeerId() == mOwnId) - { - saveData.push_back(*mit); - } - } - - // also save pending private publish keys you may have - std::map::iterator pendKeyIt; - - for(pendKeyIt = mRecvdPubKeys.begin(); pendKeyIt != mRecvdPubKeys.end(); pendKeyIt++) - { - - RsDistribGrpKey *pubKey = new RsDistribGrpKey(); - pubKey->grpId = pendKeyIt->first; - const unsigned char *keyptr = (const unsigned char *) - pendKeyIt->second->key.keyData.bin_data; - long keylen = pendKeyIt->second->key.keyData.bin_len; - - RSA *rsa_priv = d2i_RSAPrivateKey(NULL, &(keyptr), keylen); - - p3DistribSecurity::setRSAPrivateKey(pubKey->key, rsa_priv); - RSA_free(rsa_priv); - - pubKey->key.keyFlags = pendKeyIt->second->key.keyFlags; - pubKey->key.startTS = pendKeyIt->second->key.startTS; - pubKey->key.endTS = pendKeyIt->second->key.endTS; - - saveData.push_back(pubKey); - saveCleanupList.push_back(pubKey); - - } - - std::list childSaveL = childSaveList(); - std::list::iterator cit = childSaveL.begin(); - RsSerialType *childSer = createSerialiser(); - uint32_t pktSize = 0; - unsigned char *data = NULL; - - for(; cit != childSaveL.end() ; cit++) - { - RsDistribConfigData* childConfig = new RsDistribConfigData(); - - pktSize = childSer->size(*cit); - data = new unsigned char[pktSize]; - childSer->serialise(*cit, data, &pktSize); - childConfig->service_data.setBinData(data, pktSize); - delete[] data; - saveData.push_back(childConfig); - saveCleanupList.push_back(childConfig); - } - - delete childSer; - - return true; -} - -void p3GroupDistrib::saveDone() -{ - /* clean up the save List */ - std::list::iterator it; - for(it = saveCleanupList.begin(); it != saveCleanupList.end(); it++) - { - delete (*it); - } - - saveCleanupList.clear(); - - /* unlock mutex */ - distribMtx.unlock(); /****** MUTEX UNLOCKED *******/ -} - -bool p3GroupDistrib::loadList(std::list& load) -{ - std::list::iterator lit; - - /* for child config data */ - std::list childLoadL; - RsSerialType* childSer = createSerialiser(); - - for(lit = load.begin(); lit != load.end(); lit++) - { - /* decide what type it is */ - - RsDistribGrp *newGrp = NULL; - RsDistribGrpKey *newKey = NULL; - RsDistribSignedMsg *newMsg = NULL; - RsDistribConfigData* newChildConfig = NULL; - - - if ((newGrp = dynamic_cast(*lit))) - { - const std::string &gid = newGrp -> grpId; - - loadGroup(newGrp, false); - subscribeToGroup(gid, true); - mSubscribedGrp.insert(gid); - } - else if ((newKey = dynamic_cast(*lit))) - { - - // for shared keys keep - if(newKey->key.keyFlags & RSTLV_KEY_TYPE_SHARED){ - - mRecvdPubKeys.insert(std::pair( - newKey->grpId, newKey)); - mPubKeyAvailableGrpId.insert(newKey->grpId); - continue; - } - - loadGroupKey(newKey, false); - - - } - else if ((newMsg = dynamic_cast(*lit))) - { - newMsg->PeerId(mOwnId); - loadMsg(newMsg, mOwnId, false, false); /* false so it'll pushed to PendingPublish list */ - } - else if ((newChildConfig = dynamic_cast(*lit))) - { - RsItem* childConfigItem = childSer->deserialise(newChildConfig->service_data.bin_data, - &newChildConfig->service_data.bin_len); - - childLoadL.push_back(childConfigItem); - - delete newChildConfig ; - } - } - - /* no need to republish until something new comes in */ - RsStackMutex stack(distribMtx); /******* STACK LOCKED MUTEX ***********/ - - childLoadList(childLoadL); // send configurations down to child - - mGroupsRepublish = false; - delete childSer; - - return true; -} - -/************************************* p3Config *************************************/ - -/* This Streamer is used for Reading and Writing Cache Files.... - * As All the child packets are Packed, we should only need RsSerialDistrib() in it. - */ - -pqistore *p3GroupDistrib::createStore(BinInterface *bio, const std::string &src, uint32_t bioflags) -{ - RsSerialiser *rsSerialiser = new RsSerialiser(); - RsSerialType *serialType = new RsDistribSerialiser(); - rsSerialiser->addSerialType(serialType); - - pqistore *store = new pqistore(rsSerialiser, src, bio, bioflags); - - return store; -} - - -/***************************************************************************************/ -/***************************************************************************************/ - /********************** Create Content ***************************************/ -/***************************************************************************************/ -/***************************************************************************************/ - -std::string p3GroupDistrib::createGroup(std::wstring name, std::wstring desc, uint32_t flags, - unsigned char* pngImageData, uint32_t imageSize) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::createGroup()" << std::endl; - std::cerr << std::endl; -#endif - /* Create a Group */ - std::string grpId; - time_t now = time(NULL); - - /* for backup */ - std::list grpKeySet; - - /* create Keys */ - RSA *rsa_admin = RSA_generate_key(2048, 65537, NULL, NULL); - RSA *rsa_admin_pub = RSAPublicKey_dup(rsa_admin); - - RSA *rsa_publish = RSA_generate_key(2048, 65537, NULL, NULL); - RSA *rsa_publish_pub = RSAPublicKey_dup(rsa_publish); - - /* Create Group Description */ - - RsDistribGrp *newGrp = new RsDistribGrp(); - - newGrp->grpName = name; - newGrp->grpDesc = desc; - newGrp->timestamp = now; - newGrp->grpFlags = flags & (RS_DISTRIB_PRIVACY_MASK | RS_DISTRIB_AUTHEN_MASK); - newGrp->grpControlFlags = 0; - - // explicit member wise copy for grp image - if((pngImageData != NULL) && (imageSize > 0)){ - newGrp->grpPixmap.binData.bin_data = new unsigned char[imageSize]; - - memcpy(newGrp->grpPixmap.binData.bin_data, pngImageData, - imageSize*sizeof(unsigned char)); - newGrp->grpPixmap.binData.bin_len = imageSize; - newGrp->grpPixmap.image_type = RSTLV_IMAGE_TYPE_PNG; - - }else{ - newGrp->grpPixmap.binData.bin_data = NULL; - newGrp->grpPixmap.binData.bin_len = 0; - newGrp->grpPixmap.image_type = 0; - } - - - - - /* set keys */ - p3DistribSecurity::setRSAPublicKey(newGrp->adminKey, rsa_admin_pub); - newGrp->adminKey.keyFlags = RSTLV_KEY_TYPE_PUBLIC_ONLY | RSTLV_KEY_DISTRIB_ADMIN; - newGrp->adminKey.startTS = now; - newGrp->adminKey.endTS = 0; /* no end */ - grpId = newGrp->adminKey.keyId; - - - RsTlvSecurityKey publish_key; - - p3DistribSecurity::setRSAPublicKey(publish_key, rsa_publish_pub); - - publish_key.keyFlags = RSTLV_KEY_TYPE_PUBLIC_ONLY; - if (flags & RS_DISTRIB_PUBLIC) - { - publish_key.keyFlags |= RSTLV_KEY_DISTRIB_PUBLIC; - } - else - { - publish_key.keyFlags |= RSTLV_KEY_DISTRIB_PRIVATE; - } - - publish_key.startTS = now; - publish_key.endTS = now + 60 * 60 * 24 * 365 * 5; /* approx 5 years */ - - newGrp->publishKeys.keys[publish_key.keyId] = publish_key; - newGrp->publishKeys.groupId = newGrp->adminKey.keyId; - - - /************* create Key Messages (to Add Later) *********************/ - RsDistribGrpKey *adKey = new RsDistribGrpKey(); - adKey->grpId = grpId; - - p3DistribSecurity::setRSAPrivateKey(adKey->key, rsa_admin); - adKey->key.keyFlags = RSTLV_KEY_TYPE_FULL | RSTLV_KEY_DISTRIB_ADMIN; - adKey->key.startTS = newGrp->adminKey.startTS; - adKey->key.endTS = newGrp->adminKey.endTS; - - - RsDistribGrpKey *pubKey = new RsDistribGrpKey(); - pubKey->grpId = grpId; - - p3DistribSecurity::setRSAPrivateKey(pubKey->key, rsa_publish); - pubKey->key.keyFlags = RSTLV_KEY_TYPE_FULL; - if (flags & RS_DISTRIB_PUBLIC) - { - pubKey->key.keyFlags |= RSTLV_KEY_DISTRIB_PUBLIC; - } - else - { - pubKey->key.keyFlags |= RSTLV_KEY_DISTRIB_PRIVATE; - } - pubKey->key.startTS = publish_key.startTS; - pubKey->key.endTS = publish_key.endTS; - /************* create Key Messages (to Add Later) *********************/ - - - /* clean up publish_key manually -> else - * the data will get deleted... - */ - - publish_key.keyData.bin_data = NULL; - publish_key.keyData.bin_len = 0; - - newGrp->grpId = grpId; - - /************* Back up Keys *********************/ - - grpKeySet.push_back(adKey); - grpKeySet.push_back(pubKey); - - backUpKeys(grpKeySet, grpId); - - /************** Serialise and sign **************************************/ - EVP_PKEY *key_admin = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(key_admin, rsa_admin); - - newGrp->adminSignature.TlvClear(); - - RsSerialType *serialType = new RsDistribSerialiser(); - - uint32_t size = serialType->size(newGrp); - char* data = new char[size]; - - serialType->serialise(newGrp, data, &size); - - /* calc and check signature */ - EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); - - EVP_SignInit(mdctx, EVP_sha1()); - EVP_SignUpdate(mdctx, data, size); - - unsigned int siglen = EVP_PKEY_size(key_admin); - unsigned char sigbuf[siglen]; - EVP_SignFinal(mdctx, sigbuf, &siglen, key_admin); - - /* save signature */ - newGrp->adminSignature.signData.setBinData(sigbuf, siglen); - newGrp->adminSignature.keyId = grpId; - - /* clean up */ - delete serialType; - EVP_MD_CTX_destroy(mdctx); - - - /******************* clean up all Keys *******************/ - - RSA_free(rsa_admin_pub); - RSA_free(rsa_publish_pub); - RSA_free(rsa_publish); - EVP_PKEY_free(key_admin); - - /******************* load up new Group *********************/ - loadGroup(newGrp, false); - - /* add Keys to GroupInfo */ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - GroupInfo *gi = locked_getGroupInfo(grpId); - if (!gi) - { - return grpId; - } - - gi->flags |= RS_DISTRIB_SUBSCRIBED; - mGroupsRepublish = true; - - /* replace the public keys */ - locked_updateGroupAdminKey(*gi, adKey); - locked_updateGroupPublishKey(*gi, pubKey); - - delete adKey; - delete pubKey; - delete[] data; - return grpId; -} - - -bool p3GroupDistrib::backUpKeys(const std::list& keysToBackUp, std::string grpId){ - -#ifdef DISTRIB_DEBUG - std::cerr << "P3Distrib::backUpKeys() Backing up keys for grpId: " << grpId << std::endl; -#endif - - std::string filename = mKeyBackUpDir + "/" + grpId + "_" + BACKUP_KEY_FILE; - std::string filenametmp = filename + ".tmp"; - - BinInterface *bio = new BinFileInterface(filenametmp.c_str(), BIN_FLAGS_WRITEABLE); - pqistore *store = createStore(bio, mOwnId, BIN_FLAGS_NO_DELETE | BIN_FLAGS_WRITEABLE); - - std::list::const_iterator it; - bool ok = true; - - for(it=keysToBackUp.begin(); it != keysToBackUp.end(); it++){ - - ok &= store->SendItem(*it); - - } - - delete store; - - if(!RsDirUtil::renameFile(filenametmp,filename)) - { - std::string errlog; -#ifdef WIN32 - rs_sprintf(errlog, "Error %u", GetLastError()); -#else - rs_sprintf(errlog, "Error %d", errno); -#endif - RsServer::notify()->AddSysMessage(0, RS_SYS_WARNING, "File rename error", "Error while renaming file " + filename + ": got error " + errlog); - return false; - } - - return ok; -} - -bool p3GroupDistrib::restoreGrpKeys(const std::string& grpId){ - - -#ifdef DISTRIB_DEBUG - std::cerr << "p3Distrib::restoreGrpKeys() Attempting to restore private keys for grp: " - << grpId << std::endl; -#endif - - // build key directory name - std::string filename = mKeyBackUpDir + "/"+ grpId + "_" + BACKUP_KEY_FILE; - - - /* create the serialiser to load keys */ - BinInterface *bio = new BinFileInterface(filename.c_str(), BIN_FLAGS_READABLE); - pqistore *store = createStore(bio, mOwnId, BIN_FLAGS_READABLE); - - RsItem* item; - bool ok = true; - bool itemAttempted = false; - RsDistribGrpKey* key = NULL; - - RsStackMutex stack(distribMtx); - - GroupInfo* gi = locked_getGroupInfo(grpId); - - //retrieve keys from file and load to appropriate grp - while(NULL != (item = store->GetItem())){ - - itemAttempted = true; - key = dynamic_cast(item); - - if(key == NULL){ -#ifdef DISTRIB_DEBUG - std::cerr << "p3groupDistrib::restoreGrpKey() Key file / grp key item not Valid, grp: " - "\ngrpId: " << grpId << std::endl; -#endif - delete store ; - return false; - } - - if(key->key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) - ok &= locked_updateGroupAdminKey(*gi, key); - else if((key->key.keyFlags & RSTLV_KEY_DISTRIB_PRIVATE) || - (key->key.keyFlags & RSTLV_KEY_DISTRIB_PUBLIC)) - ok &= locked_updateGroupPublishKey(*gi, key); - else - ok &= false; - - } - - - - ok &= itemAttempted; - - if(ok){ - gi->flags |= RS_DISTRIB_SUBSCRIBED; - locked_notifyGroupChanged(*gi, GRP_SUBSCRIBED, false); - IndicateConfigChanged(); - mGroupsRepublish = true; - } - -#ifdef DISTRIB_DEBUG - if(!ok){ - std::cerr << "p3Distrib::restoreGrpKeys() Failed to restore private keys for grp " - << "\ngrpId: " << grpId << std::endl; - } -#endif - - delete store; - - return ok; -} - - -bool p3GroupDistrib::sharePubKey(std::string grpId, std::list& peers){ - - RsStackMutex stack(distribMtx); - - // first check that group actually exists - if(mGroups.find(grpId) == mGroups.end()){ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::sharePubKey(): Group does not exist" << std::endl; -#endif - return false; - } - - // add to pending list to be sent - mPendingPubKeyRecipients[grpId] = peers; - - return true; -} - -void p3GroupDistrib::locked_sharePubKey(){ - - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_sharePubKey() " << std::endl; -#endif - - std::map >::iterator mit; - std::list::iterator lit; - - // get list of peers that are online - std::list peersOnline; - rsPeers->getOnlineList(peersOnline); - std::list toDelete; - - /* send public key to peers online */ - - for(mit = mPendingPubKeyRecipients.begin(); mit != mPendingPubKeyRecipients.end(); mit++){ - - GroupInfo *gi = locked_getGroupInfo(mit->first); - - if(gi == NULL){ - toDelete.push_back(mit->first); // grp does not exist, stop attempting to share key for dead group - continue; - } - - // find full public key, and send to given peers - std::map::iterator kit; - for(kit = gi->publishKeys.begin(); - kit != gi->publishKeys.end(); kit++) - { - if (kit->second.type & RSTLV_KEY_TYPE_FULL) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_sharePubKey() Sharing Key: " << kit->first; - std::cerr << std::endl; -#endif - - // send keys to peers who are online - for(lit = mit->second.begin() ; lit != mit->second.end(); lit++){ - - if(std::find(peersOnline.begin(), peersOnline.end(), *lit) != peersOnline.end()){ - - /* create Key for sharing */ - RsDistribGrpKey* pubKey = new RsDistribGrpKey(getRsItemService(getType())); - - - pubKey->clear(); - pubKey->grpId = mit->first; - - RSA *rsa_priv = EVP_PKEY_get1_RSA(kit->second.key); - p3DistribSecurity::setRSAPrivateKey(pubKey->key, rsa_priv); - RSA_free(rsa_priv); - - pubKey->key.keyFlags = kit->second.type; - pubKey->key.startTS = kit->second.startTS; - pubKey->key.endTS = kit->second.endTS; - pubKey->PeerId(*lit); - std::cout << *lit << std::endl; - sendItem(pubKey); - - // remove peer from list - lit = mit->second.erase(lit); // no need to send to peer anymore - lit--; - } - } - } - } - - // if given peers have all received key(s) then stop sending for group - if(mit->second.empty()) - toDelete.push_back(mit->first); - } - - // delete pending peer list which are done with - for(lit = toDelete.begin(); lit != toDelete.end(); lit++) - mPendingPubKeyRecipients.erase(*lit); - - mLastKeyPublishTime = time(NULL); - - return; -} - - -void p3GroupDistrib::receivePubKeys(){ - - - RsItem* item; - std::string gid; - - std::map::iterator it; - std::list toDelete; - std::list::iterator sit; - - - // load received keys - while(NULL != (item = recvItem())){ - - RsDistribGrpKey* key_item = dynamic_cast(item); - - if(key_item != NULL){ - RsStackMutex stack(distribMtx); - - it = mGroups.find(key_item->grpId); - - // if group does not exist keep to see if it arrives later - if(it == mGroups.end()){ - - // make sure key is in date - if(((time_t)(key_item->key.startTS + mStorePeriod) > time(NULL)) && - (key_item->key.keyFlags & RSTLV_KEY_TYPE_FULL)){ - - // make sure keys does not exist in recieved list, then delete - if(mRecvdPubKeys.find(gid) == mRecvdPubKeys.end()){ - - // id key as shared so on loadlist sends back mRecvdPubKeys - key_item->key.keyFlags |= RSTLV_KEY_TYPE_SHARED; - mRecvdPubKeys.insert(std::pair(key_item->grpId, key_item)); - - mPubKeyAvailableGrpId.insert(key_item->grpId); - } - - }else{ - delete key_item; - } - - continue; - } - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_receiveKeys()" << std::endl; - std::cerr << "PeerId : " << key_item->PeerId() << std::endl; - std::cerr << "GrpId: " << key_item->grpId << std::endl; - std::cerr << "Got key Item" << std::endl; -#endif - if(key_item->key.keyFlags & RSTLV_KEY_TYPE_FULL){ - - gid = key_item->grpId; - - // add key if user is subscribed if not store it until user subscribes - - if(locked_updateGroupPublishKey(it->second, key_item)){ - - mPubKeyAvailableGrpId.insert(key_item->grpId); - locked_notifyGroupChanged(it->second, GRP_LOAD_KEY, false); - - // keep key if user not subscribed - if(it->second.flags & RS_DISTRIB_SUBSCRIBED){ - - delete key_item; - - }else{ - - // make sure keys does not exist in recieved list - if(mRecvdPubKeys.find(gid) == mRecvdPubKeys.end()){ - - // id key as shared so on loadlist sends back to mRecvdPubKeys - key_item->key.keyFlags |= RSTLV_KEY_TYPE_SHARED; - mRecvdPubKeys.insert(std::pair(key_item->grpId, key_item)); - - } - } - - } - } - else{ - std::cerr << "p3GroupDistrib::locked_receiveKeys():" << "Not full public key" - << "Deleting item"<< std::endl; - - delete key_item; - } - } - else{ - delete item; - } - } - - - - - RsStackMutex stack(distribMtx); - - // indicate config changed and also record the groups keys received for - if(!mRecvdPubKeys.empty()) - IndicateConfigChanged(); - - return; - - - } - - -std::string p3GroupDistrib::publishMsg(RsDistribMsg *msg, bool personalSign) -{ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishMsg()" << std::endl; - msg->print(std::cerr, 10); - std::cerr << std::endl; -#endif - - /* extract grpId */ - std::string grpId = msg->grpId; - std::string msgId; - - RsDistribSignedMsg *signedMsg = NULL; - - /* ensure Group exists */ - { /* STACK MUTEX */ - - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - GroupInfo *gi = locked_getGroupInfo(grpId); - if (!gi) - { - #ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishMsg() No Group"; - std::cerr << std::endl; - #endif - return msgId; - } - - /******************* FIND KEY ******************************/ - if (!locked_choosePublishKey(*gi)) - { - #ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishMsg() No Publish Key(1)"; - std::cerr << std::endl; - #endif - return msgId; - } - - /* find valid publish_key */ - EVP_PKEY *publishKey = NULL; - std::map::iterator kit; - kit = gi->publishKeys.find(gi->publishKeyId); - if (kit != gi->publishKeys.end()) - { - publishKey = kit->second.key; - } - - if (!publishKey) - { - #ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishMsg() No Publish Key"; - std::cerr << std::endl; - #endif - /* no publish Key */ - return msgId; - } - - /******************* FIND KEY ******************************/ - - signedMsg = new RsDistribSignedMsg(); - - RsSerialType *serialType = createSerialiser(); - uint32_t size = serialType->size(msg); - char *data = new char[size]; - serialType->serialise(msg, data, &size); - - char *out_data = NULL; - uint32_t out_size = 0; - - // encrypt data if group is private - - if(gi->grpFlags & RS_DISTRIB_ENCRYPTED){ - - EVP_PKEY * privateKey = NULL; - std::map::iterator kit; - - for(kit = gi->publishKeys.begin(); kit != gi->publishKeys.end(); kit++ ){ - // Does not allow for possibility of different keys - - if((kit->second.type & RSTLV_KEY_TYPE_FULL) && (kit->second.key->type == EVP_PKEY_RSA)){ - privateKey = kit->second.key; - break; - } - } - - if(kit == gi->publishKeys.end()){ - #ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishMsg(): Cannot find full key for encryption, " - << "grpId " << grpId - << std::endl; - #endif - return msgId; - } - - if(p3DistribSecurity::encrypt((void*&)out_data, (int&)out_size, (void*&)data, (int)size, privateKey)){ - - delete[] data; - }else{ - delete[] data; - delete signedMsg; - delete serialType; - return msgId; - } - - }else - { - out_data = data; - out_size = size; - } - - signedMsg->packet.setBinData(out_data, out_size); - - /* sign Packet */ - - /* calc and check signature */ - EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); - - EVP_SignInit(mdctx, EVP_sha1()); - EVP_SignUpdate(mdctx, out_data, out_size); - - unsigned int siglen = EVP_PKEY_size(publishKey); - unsigned char sigbuf[siglen]; - EVP_SignFinal(mdctx, sigbuf, &siglen, publishKey); - - /* save signature */ - signedMsg->publishSignature.signData.setBinData(sigbuf, siglen); - signedMsg->publishSignature.keyId = gi->publishKeyId; - - bool ok = true; - - if (personalSign) - { - unsigned int siglen = MAX_GPG_SIGNATURE_SIZE; - unsigned char sigbuf[siglen]; - if (AuthGPG::getAuthGPG()->SignDataBin(out_data, out_size, sigbuf, &siglen)) - { - signedMsg->personalSignature.signData.setBinData(sigbuf, siglen); - signedMsg->personalSignature.keyId = AuthGPG::getAuthGPG()->getGPGOwnId(); - } else { - ok = false; - } - } - - /* clean up */ - delete serialType; - EVP_MD_CTX_destroy(mdctx); - delete[] out_data; - - if (ok == false) { - delete signedMsg; - return msgId; - } - - } /* END STACK MUTEX */ - - /* extract Ids from publishSignature */ - signedMsg->msgId = p3DistribSecurity::getBinDataSign( - signedMsg->publishSignature.signData.bin_data, - signedMsg->publishSignature.signData.bin_len); - signedMsg->grpId = grpId; - signedMsg->timestamp = msg->timestamp; - - msgId = signedMsg->msgId; - - /* delete original msg */ - delete msg; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::publishMsg() Created SignedMsg:"; - std::cerr << std::endl; - signedMsg->print(std::cerr, 10); - std::cerr << std::endl; -#endif - - /* load proper - - * If we pretend it is coming from an alternative source - * it'll automatically get published with other msgs - */ - - signedMsg->PeerId(mOwnId); - loadMsg(signedMsg, mOwnId, false, false); - - /* done */ - return msgId; -} - - - - -/********************* Overloaded Functions **************************/ - - -bool p3GroupDistrib::locked_checkGroupInfo(GroupInfo &info, RsDistribGrp *newGrp) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupInfo()"; - std::cerr << std::endl; -#endif - /* groupInfo */ - - /* If adminKey is the same and - * timestamp is <= timestamp, or not an update (info edit) - * then just discard it. - */ - - if (info.grpId != newGrp->grpId) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupInfo() Failed GrpId Wrong"; - std::cerr << std::endl; -#endif - return false; - } - - if ((info.distribGroup) && (newGrp->timestamp <= info.distribGroup->timestamp)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupInfo() Group Data Old/Same"; - std::cerr << std::endl; -#endif - /* old or same info -> drop it */ - return false; - } - - /* otherwise validate it */ - return p3DistribSecurity::validateDistribGrp(newGrp); -} - - -/* return false - to cleanup (delete group) afterwards, - * true - we've kept the data - */ -bool p3GroupDistrib::locked_updateGroupInfo(GroupInfo &info, RsDistribGrp *newGrp) -{ - /* new group has been validated already - * update information. - */ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupInfo()"; - std::cerr << std::endl; -#endif - - if (info.distribGroup) - { - delete info.distribGroup; - } - - if (info.grpIcon.pngImageData != NULL){ - delete[] info.grpIcon.pngImageData; - info.grpIcon.imageSize = 0; - } - - info.distribGroup = newGrp; - - /* copy details */ - info.grpName = newGrp->grpName; - info.grpDesc = newGrp->grpDesc; - info.grpCategory = newGrp->grpCategory; - info.grpFlags = newGrp->grpFlags; - - if((newGrp->grpPixmap.binData.bin_data != NULL) && (newGrp->grpPixmap.binData.bin_len > 0)){ - info.grpIcon.pngImageData = new unsigned char[newGrp->grpPixmap.binData.bin_len]; - - memcpy(info.grpIcon.pngImageData, newGrp->grpPixmap.binData.bin_data, - newGrp->grpPixmap.binData.bin_len*sizeof(unsigned char)); - - info.grpIcon.imageSize = newGrp->grpPixmap.binData.bin_len; - }else{ - info.grpIcon.pngImageData = NULL; - info.grpIcon.imageSize = 0; - } - - /* pop already calculated */ - /* last post handled seperately */ - - locked_checkGroupKeys(info); - - /* if we are subscribed to the group -> then we need to republish */ - if (info.flags & RS_DISTRIB_SUBSCRIBED) - { - mGroupsRepublish = true; - } - - return true; -} - - -bool p3GroupDistrib::locked_editGroup(std::string grpId, GroupInfo& gi){ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_editGroup() " << grpId << std::endl; -#endif - - GroupInfo* gi_curr = locked_getGroupInfo(grpId); - - if(gi_curr == NULL){ - - std::cerr << "p3GroupDistrib::locked_editGroup() Failed, group does not exist " << grpId - << std::endl; - return false; - } - - if(!(gi_curr->flags & RS_DISTRIB_ADMIN)) - return false; - - - gi_curr->grpName = gi.grpName; - gi_curr->distribGroup->grpName = gi_curr->grpName; - gi_curr->grpDesc = gi.grpDesc; - gi_curr->distribGroup->grpDesc = gi_curr->grpDesc; - - if((gi.grpIcon.imageSize != 0) && gi.grpIcon.pngImageData != NULL){ - - if((gi_curr->distribGroup->grpPixmap.binData.bin_data != NULL) && - (gi_curr->distribGroup->grpPixmap.binData.bin_len != 0)) - gi_curr->distribGroup->grpPixmap.binData.TlvClear(); - - gi_curr->distribGroup->grpPixmap.binData.bin_data = gi_curr->grpIcon.pngImageData; - gi_curr->distribGroup->grpPixmap.binData.bin_len = gi_curr->grpIcon.imageSize; - - gi_curr->grpIcon.imageSize = gi.grpIcon.imageSize; - gi_curr->grpIcon.pngImageData = gi.grpIcon.pngImageData; - - - } - - // set new timestamp for grp - gi_curr->distribGroup->timestamp = time(NULL); - - // create new signature for group - - EVP_PKEY *key_admin = gi_curr->adminKey.key; - gi_curr->distribGroup->adminSignature.TlvClear(); - RsSerialType *serialType = new RsDistribSerialiser(); - uint32_t size = serialType->size(gi_curr->distribGroup); - char* data = new char[size]; - serialType->serialise(gi_curr->distribGroup, data, &size); - - /* calc and check signature */ - EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); - EVP_SignInit(mdctx, EVP_sha1()); - EVP_SignUpdate(mdctx, data, size); - - unsigned int siglen = EVP_PKEY_size(key_admin); - unsigned char sigbuf[siglen]; - EVP_SignFinal(mdctx, sigbuf, &siglen, key_admin); - - /* save signature */ - gi_curr->distribGroup->adminSignature.signData.setBinData(sigbuf, siglen); - gi_curr->distribGroup->adminSignature.keyId = grpId; - - mGroupsChanged = true; - gi_curr->grpChanged = true; - mGroupsRepublish = true; - - - - delete[] data; - delete serialType; - EVP_MD_CTX_destroy(mdctx); - - return true; -} - -bool p3GroupDistrib::locked_checkGroupKeys(GroupInfo &info) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys()"; - std::cerr << "GrpId: " << info.grpId; - std::cerr << std::endl; -#endif - - /* iterate through publish keys - check that they exist in distribGrp, or delete */ - RsDistribGrp *grp = info.distribGroup; - - std::list removeKeys; - std::map::iterator it; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Checking if Expanded Keys still Exist"; - std::cerr << std::endl; -#endif - - for(it = info.publishKeys.begin(); it != info.publishKeys.end(); it++) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Publish Key: " << it->first; -#endif - /* check for key in distribGrp */ - if (grp->publishKeys.keys.end() == grp->publishKeys.keys.find(it->first)) - { - /* remove publishKey */ - removeKeys.push_back(it->first); -#ifdef DISTRIB_DEBUG - std::cerr << " Old -> to Remove" << std::endl; -#endif - - } - -#ifdef DISTRIB_DEBUG - std::cerr << " Ok" << std::endl; -#endif - - } - - while(removeKeys.size() > 0) - { - std::string rkey = removeKeys.front(); - removeKeys.pop_front(); -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Removing Key: " << rkey; - std::cerr << std::endl; -#endif - - it = info.publishKeys.find(rkey); - EVP_PKEY_free(it->second.key); - info.publishKeys.erase(it); - } - - /* iterate through distribGrp list - expanding any missing keys */ - std::map::iterator dit; - for(dit = grp->publishKeys.keys.begin(); dit != grp->publishKeys.keys.end(); dit++) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Checking for New Keys: KeyId: " << dit->first; - std::cerr << std::endl; -#endif - - it = info.publishKeys.find(dit->first); - if (it == info.publishKeys.end()) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Key Missing - Expand"; - std::cerr << std::endl; -#endif - - /* create a new expanded public key */ - RSA *rsa_pub = p3DistribSecurity::extractPublicKey(dit->second); - if (!rsa_pub) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Failed to Expand Key"; - std::cerr << std::endl; -#endif - continue; - } - - GroupKey newKey; - newKey.keyId = dit->first; - newKey.type = RSTLV_KEY_TYPE_PUBLIC_ONLY | (dit->second.keyFlags & RSTLV_KEY_DISTRIB_MASK); - newKey.startTS = dit->second.startTS; - newKey.endTS = dit->second.endTS; - - newKey.key = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(newKey.key, rsa_pub); - - info.publishKeys[newKey.keyId] = newKey; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Expanded Key: " << dit->first; - std::cerr << "Key Type: " << newKey.type; - std::cerr << std::endl; - std::cerr << "Start: " << newKey.startTS; - std::cerr << std::endl; - std::cerr << "End: " << newKey.endTS; - std::cerr << std::endl; -#endif - } - } - - /* now check admin key */ - if ((info.adminKey.keyId == "") || (!info.adminKey.key)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() Must Expand AdminKey Too"; - std::cerr << std::endl; -#endif - - /* must expand admin key too */ - RSA *rsa_pub = p3DistribSecurity::extractPublicKey(grp->adminKey); - if (rsa_pub) - { - info.adminKey.keyId = grp->adminKey.keyId; - info.adminKey.type = RSTLV_KEY_TYPE_PUBLIC_ONLY & RSTLV_KEY_DISTRIB_ADMIN; - info.adminKey.startTS = grp->adminKey.startTS; - info.adminKey.endTS = grp->adminKey.endTS; - - info.adminKey.key = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(info.adminKey.key, rsa_pub); -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() AdminKey Expanded"; - std::cerr << std::endl; -#endif - } - else - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkGroupKeys() ERROR Expandng AdminKey"; - std::cerr << std::endl; -#endif - } - } - - return true; -} - - - - -bool p3GroupDistrib::locked_updateGroupAdminKey(GroupInfo &info, RsDistribGrpKey *newKey) -{ - /* so firstly - check that the KeyId matches something in the group */ - const std::string &keyId = newKey->key.keyId; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupAdminKey() grpId: " << keyId; - std::cerr << std::endl; -#endif - - - if (keyId != info.grpId) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupAdminKey() Id mismatch - ERROR"; - std::cerr << std::endl; -#endif - - return false; - } - - if (!(newKey->key.keyFlags & RSTLV_KEY_TYPE_FULL)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupAdminKey() Key not Full - Ignore"; - std::cerr << std::endl; -#endif - - /* not a full key -> ignore */ - return false; - } - - if (info.adminKey.type & RSTLV_KEY_TYPE_FULL) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupAdminKey() Already have Full Key - Ignore"; - std::cerr << std::endl; -#endif - - /* if we have full key already - ignore */ - return true; - } - - /* need to update key */ - RSA *rsa_priv = p3DistribSecurity::extractPrivateKey(newKey->key); - - if (!rsa_priv) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupAdminKey() Extract Key failed - ERROR"; - std::cerr << std::endl; -#endif - - return false; - } - - /* validate they are a matching pair */ - std::string realkeyId = p3DistribSecurity::getRsaKeySign(rsa_priv); - if ((1 != RSA_check_key(rsa_priv)) || (realkeyId != keyId)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupAdminKey() Validate Key Failed - ERROR"; - std::cerr << std::endl; -#endif - - /* clean up */ - RSA_free(rsa_priv); - return false; - } - - /* add it in */ - EVP_PKEY *evp_pkey = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(evp_pkey, rsa_priv); - - EVP_PKEY_free(info.adminKey.key); - info.adminKey.key = evp_pkey; - info.adminKey.type = RSTLV_KEY_TYPE_FULL | RSTLV_KEY_DISTRIB_ADMIN; - - info.flags |= RS_DISTRIB_ADMIN; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupAdminKey() Success"; - std::cerr << std::endl; -#endif - - return true; -} - - -bool p3GroupDistrib::locked_updateGroupPublishKey(GroupInfo &info, RsDistribGrpKey *newKey) -{ - /* so firstly - check that the KeyId matches something in the group */ - const std::string &keyId = newKey->key.keyId; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupPublishKey() grpId: " << info.grpId << " keyId: " << keyId; - std::cerr << std::endl; -#endif - - - std::map::iterator it; - it = info.publishKeys.find(keyId); - if (it == info.publishKeys.end()) - { - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupPublishKey() key not Found - Ignore"; - std::cerr << std::endl; -#endif - - /* no key -> ignore */ - return false; - } - - if (!(newKey->key.keyFlags & RSTLV_KEY_TYPE_FULL)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupPublishKey() not FullKey - Ignore"; - std::cerr << std::endl; -#endif - - /* not a full key -> ignore */ - return false; - } - - if (it->second.type & RSTLV_KEY_TYPE_FULL) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupPublishKey() already have FullKey - Ignore"; - std::cerr << std::endl; -#endif - - /* if we have full key already - ignore */ - return true; - } - - /* need to update key */ - RSA *rsa_priv = p3DistribSecurity::extractPrivateKey(newKey->key); - - if (!rsa_priv) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupPublishKey() Private Extract Failed - ERROR"; - std::cerr << std::endl; -#endif - - return false; - } - - /* validate they are a matching pair */ - std::string realkeyId = p3DistribSecurity::getRsaKeySign(rsa_priv); - if ((1 != RSA_check_key(rsa_priv)) || (realkeyId != keyId)) - { - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupPublishKey() Validate Private Key failed - ERROR"; - std::cerr << std::endl; -#endif - - /* clean up */ - RSA_free(rsa_priv); - return false; - } - - /* add it in */ - EVP_PKEY *evp_pkey = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(evp_pkey, rsa_priv); - - EVP_PKEY_free(it->second.key); - it->second.key = evp_pkey; - it->second.type &= (~RSTLV_KEY_TYPE_PUBLIC_ONLY); - it->second.type |= RSTLV_KEY_TYPE_FULL; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_updateGroupPublishKey() Success"; - std::cerr << std::endl; - std::cerr << "Key ID: " << it->first; - std::cerr << std::endl; - std::cerr << "Key Type: " << it->second.type; - std::cerr << std::endl; - std::cerr << "Start: " << it->second.startTS; - std::cerr << std::endl; - std::cerr << "End: " << it->second.endTS; - std::cerr << std::endl; -#endif - - info.flags |= RS_DISTRIB_PUBLISH; - - /* if we have updated, we are subscribed, and it is a public key */ - if ((info.flags & RS_DISTRIB_SUBSCRIBED) && - (it->second.type & RSTLV_KEY_DISTRIB_PUBLIC)) - { - mGroupsRepublish = true; - } - - return true; -} - - -bool p3GroupDistrib::locked_choosePublishKey(GroupInfo &info) -{ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_choosePublishKey()"; - std::cerr << std::endl; -#endif - time_t now = time(NULL); - - /******************* CHECK CURRENT KEY ******************************/ - /* if current key is valid -> okay */ - - std::map::iterator kit; - kit = info.publishKeys.find(info.publishKeyId); - if (kit != info.publishKeys.end()) - { - if ((kit->second.type & RSTLV_KEY_TYPE_FULL) && - (now < kit->second.endTS) && (now > kit->second.startTS)) - { - /* key is okay */ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_choosePublishKey() Current Key is Okay"; - std::cerr << std::endl; -#endif - return true; - } - } - - /******************* FIND KEY ******************************/ - std::string bestKey = ""; - time_t bestEndTime = 0; - - for(kit = info.publishKeys.begin(); kit != info.publishKeys.end(); kit++) - { - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_choosePublishKey() Found Key: "; - std::cerr << kit->first << " type: " << kit->second.type; - std::cerr << std::endl; -#endif - - if (kit->second.type & RSTLV_KEY_TYPE_FULL) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_choosePublishKey() Found FULL Key: "; - std::cerr << kit->first << " startTS: " << kit->second.startTS; - std::cerr << " endTS: " << kit->second.startTS; - std::cerr << " now: " << now; - std::cerr << std::endl; -#endif - if ((now < kit->second.endTS) && (now >= kit->second.startTS)) - { - if (kit->second.endTS > bestEndTime) - { - bestKey = kit->first; - bestEndTime = kit->second.endTS; -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_choosePublishKey() Better Key: "; - std::cerr << kit->first; - std::cerr << std::endl; -#endif - } - } - } - } - - if (bestEndTime == 0) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_choosePublishKey() No Valid Key"; - std::cerr << std::endl; -#endif - return false; - } - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_choosePublishKey() Best Key is: " << bestKey; - std::cerr << std::endl; -#endif - - info.publishKeyId = bestKey; - return true; -} - - /* deserialise RsDistribSignedMsg */ -RsDistribMsg *p3GroupDistrib::unpackDistribSignedMsg(RsDistribSignedMsg *newMsg) -{ - RsSerialType *serialType = createSerialiser(); - uint32_t size = newMsg->packet.bin_len; - RsDistribMsg *distribMsg = (RsDistribMsg *) - serialType->deserialise(newMsg->packet.bin_data, &size); - - if (distribMsg) - { - - /* transfer data that is not in the serialiser */ - distribMsg->msgId = newMsg->msgId; - - /* Full copies required ? */ - - distribMsg->publishSignature.keyId = newMsg->publishSignature.keyId; - distribMsg->publishSignature.signData.setBinData( - newMsg->publishSignature.signData.bin_data, - newMsg->publishSignature.signData.bin_len); - - distribMsg->personalSignature.keyId = newMsg->personalSignature.keyId; - distribMsg->personalSignature.signData.setBinData( - newMsg->personalSignature.signData.bin_data, - newMsg->personalSignature.signData.bin_len); - } - - delete serialType; - - return distribMsg; -} - -void p3GroupDistrib::getGrpListPubKeyAvailable(std::list& grpList) -{ - RsStackMutex stack(distribMtx); - std::set::const_iterator cit = mPubKeyAvailableGrpId.begin(); - - for(; cit != mPubKeyAvailableGrpId.end(); cit++) - grpList.push_back(*cit); - - return; -} - -bool p3GroupDistrib::locked_checkDistribMsg( - GroupInfo &/*gi*/, RsDistribMsg *msg) -{ - - /* check timestamp */ - time_t now = time(NULL); - uint32_t min = now - mStorePeriod; - uint32_t max = now + GROUP_MAX_FWD_OFFSET; - - if ((msg->timestamp < min) || (msg->timestamp > max)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_checkDistribMsg() TS out of range"; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::locked_checkDistribMsg() msg->timestamp: " << msg->timestamp; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::locked_checkDistribMsg() = " << now - msg->timestamp << " secs ago"; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::locked_checkDistribMsg() max TS: " << max; - std::cerr << std::endl; - std::cerr << "p3GroupDistrib::locked_checkDistribMsg() min TS: " << min; - std::cerr << std::endl; -#endif - /* if outside range -> remove */ - return false; - } - - /* check filters */ - - return true; -} - - - /* now update parents TS */ -bool p3GroupDistrib::locked_updateChildTS(GroupInfo &gi, RsDistribMsg *msg) -{ - /* find all parents - update timestamp */ - time_t updateTS = msg->timestamp; - msg->childTS = updateTS; - - while("" != msg->parentId) - { - std::string parentId = msg->parentId; - - std::map::iterator mit; - if (gi.msgs.end() == (mit = gi.msgs.find(parentId))) - { - /* not found - abandon (check for dummyMsgs first) */ - return locked_updateDummyChildTS(gi, parentId, updateTS); - - } - RsDistribMsg *parent = mit->second; - if ((!parent) || (parent->childTS > updateTS)) - { - /* we're too old - give up! */ - return true; - } - - /* update timestamp */ - parent->childTS = updateTS; - msg = parent; - } - return false ; -} - - - - - - - -/***** DEBUG *****/ - -void p3GroupDistrib::printGroups(std::ostream &out) -{ - /* iterate through all the groups */ - std::map::iterator git; - for(git = mGroups.begin(); git != mGroups.end(); git++) - { - out << "GroupId: " << git->first << std::endl; - out << "Group Details: " << std::endl; - out << git->second; - out << std::endl; - - std::map::iterator mit; - for(mit = git->second.msgs.begin(); - mit != git->second.msgs.end(); mit++) - { - out << "MsgId: " << mit->first << std::endl; - out << "Message Details: " << std::endl; - mit->second->print(out, 10); - out << std::endl; - } - } -} - - -std::ostream &operator<<(std::ostream &out, const GroupInfo &info) -{ - /* print Group Info */ - out << "GroupInfo: " << info.grpId << std::endl; - out << "sources [" << info.sources.size() << "]: "; - - std::list::const_iterator sit; - for(sit = info.sources.begin(); sit != info.sources.end(); sit++) - { - out << " " << *sit; - } - out << std::endl; - - out << " Message Count: " << info.msgs.size() << std::endl; - - std::string grpName(info.grpName.begin(), info.grpName.end()); - std::string grpDesc(info.grpDesc.begin(), info.grpDesc.end()); - - out << "Group Name: " << grpName << std::endl; - out << "Group Desc: " << grpDesc << std::endl; - out << "Group Flags: " << info.flags << std::endl; - out << "Group Pop: " << info.pop << std::endl; - out << "Last Post: " << info.lastPost << std::endl; - - out << "PublishKeyId:" << info.publishKeyId << std::endl; - - out << "Published RsDistribGrp: " << std::endl; - if (info.distribGroup) - { - info.distribGroup->print(out, 10); - } - else - { - out << "No RsDistribGroup Object" << std::endl; - } - - return out; -} - -void p3GroupDistrib::locked_notifyGroupChanged(GroupInfo &info, uint32_t /*flags*/, bool /*historical*/) -{ - mGroupsChanged = true; - info.grpChanged = true; -} - - -bool p3GroupDistrib::groupsChanged(std::list &groupIds) -{ - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - - /* iterate through all groups and get changed list */ - if (!mGroupsChanged) - return false; - - std::map::iterator it; - for(it = mGroups.begin(); it != mGroups.end(); it++) - { - if (it->second.grpChanged) - { - groupIds.push_back(it->first); - it->second.grpChanged = false; - } - } - - mGroupsChanged = false; - return true; -} - - - -/***************************************************************************************/ -/***************************************************************************************/ - /******************* Handle Missing Messages ***********************************/ -/***************************************************************************************/ -/***************************************************************************************/ - -/* Find missing messages */ - - - -/* LOGIC: - * - * dummy(grpId, threadId, parentId, msgId); - * - * add new msg.... - * - search for threadId. - * - if missing add thread head: dummy(grpId, threadId, NULL, threadId). - * - * - search for parentId - * - if = threadId, we just added it (ok). - * - if missing add dummy(grpId, threadId, threadId, parentId). - * - * - check for matching dummy msgId. - * - if yes, delete. - * - */ - -RsDistribDummyMsg::RsDistribDummyMsg( std::string tId, std::string pId, std::string mId, uint32_t ts) -:threadId(tId), parentId(pId), msgId(mId), timestamp(ts), childTS(ts) -{ - return; -} - -std::ostream &operator<<(std::ostream &out, const RsDistribDummyMsg &msg) -{ - out << "DummyMsg(" << msg.threadId << "," << msg.parentId << "," << msg.msgId << ")"; - return out; -} - - - -bool p3GroupDistrib::locked_CheckNewMsgDummies(GroupInfo &grp, RsDistribMsg *msg, std::string /*id*/, bool /*historical*/) -{ - std::string threadId = msg->threadId; - std::string parentId = msg->parentId; - std::string msgId = msg->msgId; - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies(grpId:" << grp.grpId << ", threadId: " << threadId; - std::cerr << ", parentId:" << parentId << ", msgId: " << msgId << ")"; - std::cerr << std::endl; -#endif - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies() Pre Printout"; - std::cerr << std::endl; - locked_printDummyMsgs(grp); -#endif - - - /* search for threadId */ - if (threadId != "") - { - std::map::iterator tit = grp.msgs.find(threadId); - - if (tit == grp.msgs.end()) // not there! - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies() No ThreadId Msg, Adding DummyMsg"; - std::cerr << std::endl; -#endif - locked_addDummyMsg(grp, threadId, "", threadId, msg->timestamp); - } - else - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies() Found ThreadId Msg"; - std::cerr << std::endl; -#endif - } - } - - if (parentId != "") - { - /* search for parentId */ - std::map::iterator pit = grp.msgs.find(parentId); - - if (pit == grp.msgs.end()) // not there! - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies() No ParentId Msg, Adding DummyMsg"; - std::cerr << std::endl; -#endif - locked_addDummyMsg(grp, threadId, threadId, parentId, msg->timestamp); - } - else - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies() Found ParentId Msg"; - std::cerr << std::endl; -#endif - } - } - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies() Checking for DummyMsg"; - std::cerr << std::endl; -#endif - - /* remove existing dummy */ - locked_clearDummyMsg(grp, msgId); - - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_CheckNewMsgDummies() Post Printout"; - std::cerr << std::endl; - locked_printDummyMsgs(grp); -#endif - - return true; -} - -bool p3GroupDistrib::locked_addDummyMsg(GroupInfo &grp, std::string threadId, std::string parentId, std::string msgId, uint32_t ts) -{ -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_addDummyMsg(grpId:" << grp.grpId << ", threadId: " << threadId; - std::cerr << ", parentId:" << parentId << ", msgId: " << msgId << ")"; - std::cerr << std::endl; -#endif - - if (msgId == "") - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_addDummyMsg() ERROR not adding empty MsgId"; - std::cerr << std::endl; -#endif - return false; - } - - /* search for the msg Id */ - std::map::iterator dit = grp.dummyMsgs.find(msgId); - - if (dit == grp.dummyMsgs.end()) // not there! - { - grp.dummyMsgs[msgId] = RsDistribDummyMsg(threadId, parentId, msgId, ts); - - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_addDummyMsg() Adding Dummy Msg"; - std::cerr << std::endl; -#endif - } - else - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_addDummyMsg() Dummy Msg already there: " << dit->second; - std::cerr << std::endl; -#endif - } - - locked_updateDummyChildTS(grp, parentId, ts); // NOTE both ChildTS functions should be merged. - return true; -} - -bool p3GroupDistrib::locked_clearDummyMsg(GroupInfo &grp, std::string msgId) -{ -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_clearDummyMsg(grpId:" << grp.grpId << ", msgId: " << msgId << ")"; - std::cerr << std::endl; -#endif - - /* search for the msg Id */ - std::map::iterator dit = grp.dummyMsgs.find(msgId); - if (dit != grp.dummyMsgs.end()) - { - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_clearDummyMsg() Erasing Dummy Msg: " << dit->second; - std::cerr << std::endl; -#endif - - grp.dummyMsgs.erase(dit); - } - else - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_clearDummyMsg() Msg not found"; - std::cerr << std::endl; -#endif - } - return true; -} - - - - /* now update parents TS */ -/* NB: it is a hack to have seperate updateChildTS functions for msgs and dummyMsgs, - * this need to be combined (do when we add a parentId index.) - */ - -bool p3GroupDistrib::locked_updateDummyChildTS(GroupInfo &gi, std::string parentId, time_t updateTS) -{ - while("" != parentId) - { - std::map::iterator mit; - if (gi.dummyMsgs.end() == (mit = gi.dummyMsgs.find(parentId))) - { - /* not found - abandon */ - return true; - } - RsDistribDummyMsg *parent = &(mit->second); - if (parent->childTS > updateTS) - { - /* we're too old - give up! */ - return true; - } - - /* update timestamp */ - parent->childTS = updateTS; - parentId = parent->parentId; - } - return false ; -} - - -bool p3GroupDistrib::locked_printAllDummyMsgs() -{ -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_printAllDummyMsg()"; - std::cerr << std::endl; -#endif - std::map::iterator it; - for(it = mGroups.begin(); it != mGroups.end(); it++) - { - locked_printDummyMsgs(it->second); - } - return true ; -} - - - -bool p3GroupDistrib::locked_printDummyMsgs(GroupInfo &grp) -{ -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_printDummyMsg(grpId:" << grp.grpId << ")"; - std::cerr << std::endl; -#endif - - /* search for the msg Id */ - std::map::iterator dit; - for(dit = grp.dummyMsgs.begin(); dit != grp.dummyMsgs.end(); dit++) - { - std::cerr << dit->second; - std::cerr << std::endl; - } - return true; -} - - -/***** These Functions are used by the children classes to access the dummyData - ****/ - -bool p3GroupDistrib::getDummyParentMsgList(const std::string& grpId, const std::string& pId, std::list &msgIds) -{ - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::getDummyParentMsgList(grpId:" << grpId << "," << pId << ")"; - std::cerr << std::endl; -#endif - - processCacheOptReq(grpId); - - RsStackMutex stack(distribMtx); /************* STACK MUTEX ************/ - - - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::getDummyParentMsgList() Group Not Found"; - std::cerr << std::endl; -#endif - return false; - } - - std::map::iterator mit; - - for(mit = git->second.dummyMsgs.begin(); mit != git->second.dummyMsgs.end(); mit++) - { - if (mit->second.parentId == pId) - { - msgIds.push_back(mit->first); - } - } - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::getDummyParentMsgList() found " << msgIds.size() << " msgs"; - std::cerr << std::endl; -#endif - return true; -} - - -RsDistribDummyMsg *p3GroupDistrib::locked_getGroupDummyMsg(const std::string& grpId, const std::string& msgId) -{ - -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_getGroupDummyMsg(grpId:" << grpId << "," << msgId << ")"; - std::cerr << std::endl; -#endif - /************* ALREADY LOCKED ************/ - std::map::iterator git; - if (mGroups.end() == (git = mGroups.find(grpId))) - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_getGroupDummyMsg() Group not found"; - std::cerr << std::endl; -#endif - return NULL; - } - - std::map::iterator dit; - if (git->second.dummyMsgs.end() == (dit = git->second.dummyMsgs.find(msgId))) - { -#ifdef DISTRIB_DUMMYMSG_DEBUG - std::cerr << "p3GroupDistrib::locked_getGroupDummyMsg() Msg not found"; - std::cerr << std::endl; -#endif - return NULL; - } - - return &(dit->second); -} - diff --git a/libretroshare/src/distrib/p3distrib.h b/libretroshare/src/distrib/p3distrib.h deleted file mode 100644 index 2e059789f..000000000 --- a/libretroshare/src/distrib/p3distrib.h +++ /dev/null @@ -1,756 +0,0 @@ -/* - * libretroshare/src/distrib: p3distrib.h - * - * - * Copyright 2004-2011 by Robert Fernie. - * 2010-2011 Christopher Evi-Parker - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef P3_GENERIC_DISTRIB_HEADER -#define P3_GENERIC_DISTRIB_HEADER - -#include "pqi/pqi.h" -#include "pqi/pqistore.h" -#include "pqi/p3cfgmgr.h" -#include "services/p3service.h" -#include "dbase/cachestrapper.h" -#include "serialiser/rsdistribitems.h" - -#include -#include - -#include -#include - -/* - * Group Messages.... - * - * Forums / Channels / Blogs... - * - * - * Plan. - * - * (1) First create basic structures .... algorithms. - * - * (2) integrate with Cache Source/Store for data transmission. - * (3) integrate with Serialiser for messages - * (4) bring over the final key parts from existing p3channel. - */ - -const uint32_t GROUP_MAX_FWD_OFFSET = (60 * 60 * 24 * 2); /* 2 Days */ - - -/* - * A data structure to store dummy (missing) msgs. - * They are added to the GroupInfo if there is a missing parent Msg of thread Msg - * Basic Logic is: - * - */ - -class RsDistribDummyMsg -{ - public: - RsDistribDummyMsg( std::string tId, std::string pId, std::string mId, uint32_t ts); - RsDistribDummyMsg() { return; } - std::string threadId; - std::string parentId; - std::string msgId; - - uint32_t timestamp; - time_t childTS; /* timestamp of most recent child */ -}; - - -//! for storing group keys to members of a group -/*! - * This key but be of many types, including private/public publish key, or admin prite key for group - * @see p3GroupDistrib - */ -class GroupKey -{ - public: - - GroupKey() - :type(0), startTS(0), endTS(0), key(NULL) { return; } - - uint32_t type; /// whether key is full or public - std::string keyId; - time_t startTS, endTS; - EVP_PKEY *key; /// actual group key in evp format -}; - -//! used to store group picture -/*! - * ensures use of png image format - * @see p3GroupDistrib - */ -class GroupIcon{ -public: - GroupIcon(): pngImageData(NULL), imageSize(0) { - return; - } - - ~GroupIcon(){ - - if((pngImageData != NULL) && (imageSize > 0)) - delete[] pngImageData; - - return; - } - - unsigned char* pngImageData; /// pointer to image data in png format - int imageSize; -}; - -//! used by p3groupDistrib to store mirror info found in rsDistribGroup (i.e. messages, posts, etc) -/*! - * used by p3Groudistrib to store group info, also used to communicate group information - * to p3groupdistrib inherited classes. contain - * @see rsDistribGroup - */ -class GroupInfo -{ - public: - - GroupInfo() - :distribGroup(NULL), grpFlags(0), pop(0), lastPost(0), flags(0), grpChanged(false) - { - return; - } - virtual ~GroupInfo() ; - - std::string grpId; /// the group id - RsDistribGrp *distribGroup; /// item which contains further information on group - - std::list sources; - std::map msgs; - std::map dummyMsgs; // dummyMsgs. - - /***********************************/ - - /* Copied from DistribGrp */ - std::wstring grpName; - std::wstring grpDesc; /// group description - std::wstring grpCategory; - uint32_t grpFlags; /// PRIVACY & AUTHENTICATION - - - uint32_t pop; /// popularity sources.size() - time_t lastPost; /// modded as msgs added - - /***********************************/ - - uint32_t flags; /// PUBLISH, SUBSCRIBE, ADMIN - - - std::string publishKeyId; /// current active Publish Key - std::map publishKeys; - - GroupKey adminKey; - - - GroupIcon grpIcon; - /* NOT USED YET */ - - std::map decrypted_msg_cache; /// stores a cache of messages that have been decrypted - - bool publisher, allowAnon, allowUnknown; - bool subscribed, listener; - - uint32_t type; - - /// FLAG for Client - set if changed - bool grpChanged; -}; - - - -std::ostream &operator<<(std::ostream &out, const GroupInfo &info); - -//! information on what cache stores group info -/*! - * This can refer to idividual cache message, data etc - */ -class GroupCache -{ - public: - - std::string filename; - time_t start, end; - uint16_t cacheSubId; /// used to resolve complete cache id -}; - - /* Flags for locked_notifyGroupChanged() ***/ - -const uint32_t GRP_NEW_UPDATE = 0x0001; -const uint32_t GRP_UPDATE = 0x0002; -const uint32_t GRP_LOAD_KEY = 0x0003; -const uint32_t GRP_NEW_MSG = 0x0004; -const uint32_t GRP_SUBSCRIBED = 0x0005; -const uint32_t GRP_UNSUBSCRIBED = 0x0006; - - -typedef std::map > CacheOptData; - - -//! Cache based service to implement group messaging -/*! - * - * Group Description: - * - * Master Public/Private Key: (Admin Key) used to control - * Group Name/Description/Icon. - * Filter Lists. - * Publish Keys. - * - * Publish Keys. - * TimeStore Length determined by inheriting class - * - * Everyone gets: - * Master Public Key. - * Publish Public Keys. - * blacklist, or whitelist filter. (Only useful for Non-Anonymous groups) - * Name, Desc, - * etc. - * - * Admins get Master Private Key. - * Publishers get Publish Private Key. - * - Channels only some get publish key. - * - Forums everyone gets publish private key. - * - * Group id is the public admin keys id - * - */ - -/* - * To Handle Cache Data Loading.... we want to be able to seperate Historical - * from new data (primarily for the gui's benefit). - * to do this we have a mHistoricalCaches flag, which is automatically raised at startup, - * and a function is called to cancel it (HistoricalCachesDone()). - */ - -class CacheDataPending -{ - public: - - CacheDataPending(const RsCacheData &data, bool local, bool historical); - RsCacheData mData; - bool mLocal; - bool mHistorical; -}; - -class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, public p3ThreadedService -{ - public: - - p3GroupDistrib(uint16_t subtype, - CacheStrapper *cs, CacheTransfer *cft, - std::string sourcedir, std::string storedir, std::string keyBackUpDir, - uint32_t configId, - uint32_t storePeriod, uint32_t pubPeriod); - - virtual ~p3GroupDistrib() ; - -/***************************************************************************************/ -/******************************* CACHE SOURCE / STORE Interface ************************/ -/***************************************************************************************/ -/* TO FINISH */ - - public: - - virtual bool loadLocalCache(const RsCacheData &data); /// overloaded from Cache Source - virtual int loadCache(const RsCacheData &data); /// overloaded from Cache Store - - - /* From RsThread */ - virtual void run(); /* called once the thread is started */ - - void HistoricalCachesDone(); // called when Stored Caches have been added to Pending List. - - - private: - - // derived from CacheSource - virtual bool isPeerAcceptedAsCacheReceiver(const std::string& ssl_id) ; - // derived from CacheStore - virtual bool isPeerAcceptedAsCacheProvider(const std::string& ssl_id) ; - - /* these lists are filled by the overloaded fns... then cleared by the thread */ - bool mHistoricalCaches; // initially true.... falsified by HistoricalCachesDone() - std::list mPendingCaches; - - /* top level load */ - int loadAnyCache(const RsCacheData &data, bool local, bool historical); - - /* load cache files */ - void loadFileGroups(const std::string &filename, const std::string &src, bool local, bool historical); - void loadFileMsgs(const std::string &filename, const RsCacheData& , bool local, bool historical); - bool backUpKeys(const std::list &keysToBackUp, std::string grpId); - void locked_sharePubKey(); - - /*! - * Attempt to load public key from recvd list if it exists for grpId - * @param grpId the id for the group for which private publish key is wanted - */ - bool attemptPublishKeysRecvd(); - - - - - /*! - * Simply load cache opt messages - * @param data - */ - void loadCacheOptMsgs(const RsCacheData& data, const std::string& grpId); - - protected: - - /* load cache msgs */ - - /*! - * processes cache opt request by loading data for group - * @param grpId the group to process request for - * @return false if group does not exist - */ - bool processCacheOptReq(const std::string &grpId); - - /*! - * msg is loaded to its group and republished, - * msg decrypted if grp is private - * @param msg msg to loaded - * @param src src of msg (peer id) - * @param local is this a local cache msg (your msg) - */ - bool loadMsg(RsDistribSignedMsg *msg, const std::string &src, bool local, bool historical); - - /*! - * msg is loaded to its group and republished, - * msg decrypted if grp is private - * @param msg msg to loaded - * @param src src of msg (peer id) - * @param local is this a local cache msg (your msg) - */ - bool locked_loadMsg(RsDistribSignedMsg *newMsg, const std::string &src, bool local, bool historical); - - /*! - * adds newgrp to grp set, GroupInfo type created and stored - * @param newGrp grp to be added - */ - bool loadGroup(RsDistribGrp *newGrp, bool historical); - - /*! - * Adds new keys dependent on whether it is an admin or publish key - * on return resource pointed to by newKey should be considered invalid - * @param newKey key to be added - * @return if key is loaded to group or stored return true - */ - bool loadGroupKey(RsDistribGrpKey *newKey, bool historical); - - - -/***************************************************************************************/ -/***************************************************************************************/ - -/***************************************************************************************/ -/**************************** Create Content *******************************************/ -/***************************************************************************************/ -/* TO FINISH */ - - public: - - /*! - * This create a distributed grp which is sent via cache system to connected peers - * @param name name of the group created - * @param desc description of the group - * @param flags privacy flag - * @param pngImageData pointer to image data, data is copied - * @param imageSize size of the image passed - * @return id of the group - */ - std::string createGroup(std::wstring name, std::wstring desc, uint32_t flags, unsigned char *pngImageData, uint32_t imageSize); - - /*! - * msg is packed into a signed message (and encrypted msg grp is private) and then sent via cache system to connnected peers - * @param msg - * @param personalSign whether to personal to sign image (this is done using gpg cert) - * @return the msg id - */ - std::string publishMsg(RsDistribMsg *msg, bool personalSign); - - /*! - * note: call back to locked_eventDuplicateMSg is made on execution - * @param grpId id of group to subscribe to - * @param subscribe true to subscribe and vice versa - * @return - */ - bool subscribeToGroup(const std::string &grpId, bool subscribe); - - - - /***************************************************************************************/ - /***************************************************************************************/ - - /***************************************************************************************/ - /****************************** Access Content ***************************************/ - /***************************************************************************************/ - - public: - - /*! - * get Group Lists - */ - bool getAllGroupList(std::list &grpids); - bool getSubscribedGroupList(std::list &grpids); - bool getPublishGroupList(std::list &grpids); - - /*! - * - * @param popMin lower limit for a grp's populairty in grpids - * @param popMax upper limit for a grp's popularity in grpids - * @param grpids grpids of grps which adhere to upper and lower limit of popularity - * @return nothing returned - */ - void getPopularGroupList(uint32_t popMin, uint32_t popMax, std::list &grpids); - - - /* get Msg Lists */ - bool getAllMsgList(const std::string& grpId, std::list &msgIds); - bool getParentMsgList(const std::string& grpId, const std::string& pId, std::list &msgIds); - bool getTimePeriodMsgList(const std::string& grpId, uint32_t timeMin, - uint32_t timeMax, std::list &msgIds); - - - GroupInfo *locked_getGroupInfo(const std::string& grpId); - RsDistribMsg *locked_getGroupMsg(const std::string& grpId, const std::string& msgId); - - /*! - * for retrieving the grpList for which public keys are available - */ - void getGrpListPubKeyAvailable(std::list& grpList); - - /* Filter Messages */ - -/***************************************************************************************/ -/***************************** Event Feedback ******************************************/ -/***************************************************************************************/ - - protected: - /*! - * root version (p3Distrib::) of this function must be called - */ - virtual void locked_notifyGroupChanged(GroupInfo &info, uint32_t flags, bool historical); - - /*! - * client (inheriting class) should use this to determing behaviour of - * their service when a duplicate msg is found - * @param group should be called when duplicate message loaded - * @param the duplicate message - * @param id - * @param historical: is this msg from an historical cache - * @return successfully executed or not - */ - virtual bool locked_eventDuplicateMsg(GroupInfo *, RsDistribMsg *, const std::string& id, bool historical) = 0; - - /*! - * Inheriting class should implement this as a response to a new msg arriving - * @param - * @param - * @param id src of msg (peer id) - * @param historical: is this msg from an historical cache - * @return - */ - virtual bool locked_eventNewMsg(GroupInfo *, RsDistribMsg *, const std::string& id, bool historical) = 0; - -/***************************************************************************************/ -/********************************* p3Config ********************************************/ -/***************************************************************************************/ -/* TO FINISH */ - - protected: - - virtual RsSerialiser *setupSerialiser(); - virtual bool saveList(bool &cleanup, std::list& saveList); - virtual void saveDone(); - virtual bool loadList(std::list& load); - - /*! - * called by top class, child can use to save configs - */ - virtual std::list childSaveList() = 0; - - /*! - * called by top class, child can use to load configs - */ - virtual bool childLoadList(std::list& configSaves) = 0; - -/***************************************************************************************/ -/***************************************************************************************/ - - public: - - virtual int tick(); /* overloaded form pqiService */ - -/***************************************************************************************/ -/**************************** Publish Content ******************************************/ -/***************************************************************************************/ -/* TO FINISH */ - protected: - - /* create/mod cache content */ - - /*! - * adds msg to pending msg map - * @param msg a signed message by peer - */ - void locked_toPublishMsg(RsDistribSignedMsg *msg); - - /*! - * adds pending msg - */ - void publishPendingMsgs(); - - /*! - * sends created groups to cache, to be passed to cache listeners - */ - void publishDistribGroups(); - - /*! - * removes old caches based on store period (anything that has been in local cache longer - * than the store period is deleted - * @param now the current time when method is called - */ - void clear_local_caches(time_t now); - - /*! - * assumes RsDistribMtx is locked when call is made - */ - void locked_publishPendingMsgs(); - - /*! - * @return cache sub id - */ - uint16_t locked_determineCacheSubId(); - - /** - * grp keys are backed up when a grp is created this allows user to retrieve lost keys in case config saving fails - * @param grpId the grpId id for which backup keys should be restored - * @return false if failed and vice versa - */ - virtual bool restoreGrpKeys(const std::string& grpId); /// restores a group keys from backup - - /** - * Allows user to send keys to a list of peers - * @param grpId the group for which to share public keys - * @param peers The peers to which public keys should be sent - */ - virtual bool sharePubKey(std::string grpId, std::list& peers); - - /** - * Attempt to receive publication keys - */ - virtual void receivePubKeys(); - - /** - * Allows group admin(s) to change group icon, description and name - *@param grpId group id - *@param gi the changes to grp name, icon, and description should be reflected here - */ - virtual bool locked_editGroup(std::string grpId, GroupInfo& gi); - - - /***************************************************************************************/ - /***************************************************************************************/ - - /***************************************************************************************/ - /*************************** Overloaded Functions **************************************/ - /***************************************************************************************/ - - /*! - * Overloaded by inherited classes to Pack/UnPack their messages - * @return inherited class's serialiser - */ - virtual RsSerialType *createSerialiser() = 0; - - /*! Used to Create/Load Cache Files only - * @param bio binary i/o - * @param src peer id from which write/read content originates - * @param bioflags read write permision for bio - * @return pointer to pqistore instance - */ - virtual pqistore *createStore(BinInterface *bio, const std::string &src, uint32_t bioflags); - - virtual bool locked_checkGroupInfo(GroupInfo &info, RsDistribGrp *newGrp); - virtual bool locked_updateGroupInfo(GroupInfo &info, RsDistribGrp *newGrp); - virtual bool locked_checkGroupKeys(GroupInfo &info); - - /*! - * @param info group for which admin key will be added to - * @param newKey admin key - * @return true if key successfully added - */ - virtual bool locked_updateGroupAdminKey(GroupInfo &info, RsDistribGrpKey *newKey); - - - /*! - * @param info group for which publish key will be added to - * @param newKey publish key - * @return true if publish key successfully added - */ - virtual bool locked_updateGroupPublishKey(GroupInfo &info, RsDistribGrpKey *newKey); - - /*! - * Use this to retrieve packed message from a signed message - * @param newMsg signed message - * @return pointer to unpacked msg - */ - virtual RsDistribMsg* unpackDistribSignedMsg(RsDistribSignedMsg *newMsg); - - - /*! - * message is checked to see if it is in a valid time range - * @param info - * @param msg message to be checked - * @return false if msg is outside correct time range - */ - virtual bool locked_checkDistribMsg(GroupInfo &info, RsDistribMsg *msg); - - /*! - * chooses the best publish key based on it being full and latest - * @param info group to choose publish key - * @return true if a publish key could be found - */ - virtual bool locked_choosePublishKey(GroupInfo &info); - - -//virtual RsDistribGrp *locked_createPublicDistribGrp(GroupInfo &info); -//virtual RsDistribGrp *locked_createPrivateDistribGrp(GroupInfo &info); - - -/***************************************************************************************/ -/***************************** Utility Functions ***************************************/ -/***************************************************************************************/ -/* TO FINISH */ - - /* utilities */ - std::string HashRsItem(const RsItem *item); - bool locked_updateChildTS(GroupInfo &gi, RsDistribMsg *msg); - -/***************************************************************************************/ -/***************************************************************************************/ - -/***************************************************************************************/ -/***************************** Utility Functions ***************************************/ -/***************************************************************************************/ - public: - - void printGroups(std::ostream &out); - - /*! - * returns list of ids for group caches that have changed - */ - bool groupsChanged(std::list &groupIds); - -/***************************************************************************************/ -/***************************************************************************************/ - - -/***************************************************************************************/ -/**************************** DummyMsgs Functions **************************************/ -/***************************************************************************************/ - public: - -bool locked_CheckNewMsgDummies(GroupInfo &info, RsDistribMsg *msg, std::string id, bool historical); -bool locked_addDummyMsg(GroupInfo &info, std::string threadId, std::string parentId, std::string msgId, uint32_t ts); -bool locked_clearDummyMsg(GroupInfo &info, std::string msgId); -bool locked_updateDummyChildTS(GroupInfo &gi, std::string parentId, time_t updateTS); // NOTE MUST BE MERGED WITH nromal version. - -bool locked_printAllDummyMsgs(); -bool locked_printDummyMsgs(GroupInfo &info); - - /* access the dummy msgs */ -bool getDummyParentMsgList(const std::string& grpId, const std::string& pId, std::list &msgIds); -RsDistribDummyMsg *locked_getGroupDummyMsg(const std::string& grpId, const std::string& msgId); - - - /* key cache functions - we use .... (not overloaded) - */ - - /* storage */ - protected: - - RsMutex distribMtx; /// Protects all class atrributes - std::string mOwnId; /// rs peer id - - private: - - std::list mLocalCaches; - std::map mGroups; - uint32_t mStorePeriod, mPubPeriod; - - /* Message Publishing */ - std::list mPendingPublish; - time_t mLastPublishTime; - std::map mLocalCacheTs; - uint16_t mMaxCacheSubId; - - bool mGroupsChanged; - bool mGroupsRepublish; - - std::list saveCleanupList; /* TEMPORARY LIST WHEN SAVING */ - std::string mKeyBackUpDir; - const std::string BACKUP_KEY_FILE; - - std::map mRecvdPubKeys; /// full publishing keys received from users - std::map > mPendingPubKeyRecipients; /// peers to receive publics key for a given grp - std::set mPubKeyAvailableGrpId; // groups id for which public keys are available - time_t mLastKeyPublishTime, mLastRecvdKeyTime; - - - /**** cache opt ****/ - - /* - * 1. when rs starts it loads only subscribed groups - * 2. and for unsubscribed groups these are store with their grp to cache mappings - * 3. when user clicks on a group this activates process cache which loads cache for only that group - * - */ - - /// stores map of grp to cache mapping - CacheOptData mGrpCacheMap; - - /// group subscribed to at start of rs - std::set mSubscribedGrp; - - /// unsubscribed groups that are already loaded - std::set mCacheOptLoaded; - - /// current exception group - std::string mCurrGrpException; - - - - -}; - - -/***************************************************************************************/ -/***************************************************************************************/ - -#endif // P3_GENERIC_DISTRIB_HEADER diff --git a/libretroshare/src/distrib/p3distribsecurity.cc b/libretroshare/src/distrib/p3distribsecurity.cc deleted file mode 100644 index 67f9e330a..000000000 --- a/libretroshare/src/distrib/p3distribsecurity.cc +++ /dev/null @@ -1,478 +0,0 @@ -/* - * libretroshare/src/distrib: p3distribverify.cc - * - * - * Copyright 2008-2010 by Robert Fernie - * 2011 Christopher Evi-Parker - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "p3distribsecurity.h" -#include "pqi/authgpg.h" -#include "retroshare/rsdistrib.h" -#include "retroshare/rspeers.h" - -p3DistribSecurity::p3DistribSecurity() -{ -} - -p3DistribSecurity::~p3DistribSecurity() -{ -} - -RSA *p3DistribSecurity::extractPublicKey(RsTlvSecurityKey& key) -{ - const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data; - long keylen = key.keyData.bin_len; - - /* extract admin key */ - RSA *rsakey = d2i_RSAPublicKey(NULL, &(keyptr), keylen); - - return rsakey; -} - - -bool p3DistribSecurity::validateDistribSignedMsg(GroupInfo & info, RsDistribSignedMsg *newMsg) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg()"; - std::cerr << std::endl; - std::cerr << "GroupInfo -> distribGrp:"; - std::cerr << std::endl; - info.distribGroup->print(std::cerr, 10); - std::cerr << std::endl; - std::cerr << "RsDistribSignedMsg: "; - std::cerr << std::endl; - newMsg->print(std::cerr, 10); - std::cerr << std::endl; -#endif - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() publish KeyId: " << newMsg->publishSignature.keyId << std::endl; - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() personal KeyId: " << newMsg->personalSignature.keyId << std::endl; -#endif - - /********************* check signature *******************/ - - /* find the right key */ - RsTlvSecurityKeySet &keyset = info.distribGroup->publishKeys; - - std::map::iterator kit; - kit = keyset.keys.find(newMsg->publishSignature.keyId); - - if (kit == keyset.keys.end()) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() Missing Publish Key"; - std::cerr << std::endl; -#endif - return false; - } - - /* check signature timeperiod */ - if ((newMsg->timestamp < kit->second.startTS) || - (newMsg->timestamp > kit->second.endTS)) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() TS out of range"; - std::cerr << std::endl; -#endif - return false; - } - - /* decode key */ - const unsigned char *keyptr = (const unsigned char *) kit->second.keyData.bin_data; - long keylen = kit->second.keyData.bin_len; - unsigned int siglen = newMsg->publishSignature.signData.bin_len; - unsigned char *sigbuf = (unsigned char *) newMsg->publishSignature.signData.bin_data; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() Decode Key"; - std::cerr << " keylen: " << keylen << " siglen: " << siglen; - std::cerr << std::endl; -#endif - - /* extract admin key */ - RSA *rsakey = d2i_RSAPublicKey(NULL, &(keyptr), keylen); - - if (!rsakey) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg()"; - std::cerr << " Invalid RSA Key"; - std::cerr << std::endl; - - unsigned long err = ERR_get_error(); - std::cerr << "RSA Load Failed .... CODE(" << err << ")" << std::endl; - std::cerr << ERR_error_string(err, NULL) << std::endl; - - kit->second.print(std::cerr, 10); -#endif - } - - - EVP_PKEY *signKey = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(signKey, rsakey); - - /* calc and check signature */ - EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); - - EVP_VerifyInit(mdctx, EVP_sha1()); - EVP_VerifyUpdate(mdctx, newMsg->packet.bin_data, newMsg->packet.bin_len); - int signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey); - - /* clean up */ - EVP_PKEY_free(signKey); - EVP_MD_CTX_destroy(mdctx); - - /* now verify Personal signature */ - if ((signOk == 1) && ((info.grpFlags & RS_DISTRIB_AUTHEN_MASK) & RS_DISTRIB_AUTHEN_REQ)) - { - unsigned int personalsiglen = - newMsg->personalSignature.signData.bin_len; - unsigned char *personalsigbuf = (unsigned char *) - newMsg->personalSignature.signData.bin_data; - - RsPeerDetails signerDetails; - std::string gpg_fpr; - if (AuthGPG::getAuthGPG()->getGPGDetails(newMsg->personalSignature.keyId, signerDetails)) - { - gpg_fpr = signerDetails.fpr; - } - - bool gpgSign = AuthGPG::getAuthGPG()->VerifySignBin( - newMsg->packet.bin_data, newMsg->packet.bin_len, - personalsigbuf, personalsiglen, gpg_fpr); - if (gpgSign) { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() Success for gpg signature." << std::endl; -#endif - signOk = 1; - } else { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() Fail for gpg signature." << std::endl; -#endif - signOk = 0; - } - } - - if (signOk == 1) - { -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() Signature OK"; - std::cerr << std::endl; -#endif - return true; - } - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::locked_validateDistribSignedMsg() Signature invalid"; - std::cerr << std::endl; -#endif - - return false; -} - - - -std::string p3DistribSecurity::getBinDataSign(void *data, int len) -{ - unsigned char *tmp = (unsigned char *) data; - - // copy first CERTSIGNLEN bytes... - if (len > CERTSIGNLEN) - { - len = CERTSIGNLEN; - } - - std::string id; - for(uint32_t i = 0; i < CERTSIGNLEN; i++) - { - rs_sprintf_append(id, "%02x", (uint16_t) (((uint8_t *) (tmp))[i])); - } - - return id; -} - - - -bool p3DistribSecurity::encrypt(void *& out, int & outlen, const void *in, int inlen, EVP_PKEY *privateKey) -{ - - -#ifdef DISTRIB_DEBUG - std::cerr << "p3DistribSecurity::encrypt() " << std::endl; -#endif - - RSA *rsa_publish_pub = NULL; - EVP_PKEY *public_key = NULL; - - RSA* rsa_publish = EVP_PKEY_get1_RSA(privateKey); - rsa_publish_pub = RSAPublicKey_dup(rsa_publish); - - - if(rsa_publish_pub != NULL){ - public_key = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(public_key, rsa_publish_pub); - }else{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3DistribSecurity(): Could not generate publish key " << grpId - << std::endl; -#endif - return false; - } - - EVP_CIPHER_CTX ctx; - int eklen, net_ekl; - unsigned char *ek; - unsigned char iv[EVP_MAX_IV_LENGTH]; - EVP_CIPHER_CTX_init(&ctx); - int out_currOffset = 0; - int out_offset = 0; - - int max_evp_key_size = EVP_PKEY_size(public_key); - ek = (unsigned char*)malloc(max_evp_key_size); - const EVP_CIPHER *cipher = EVP_aes_128_cbc(); - int cipher_block_size = EVP_CIPHER_block_size(cipher); - int size_net_ekl = sizeof(net_ekl); - - int max_outlen = inlen + cipher_block_size + EVP_MAX_IV_LENGTH + max_evp_key_size + size_net_ekl; - - // intialize context and send store encrypted cipher in ek - if(!EVP_SealInit(&ctx, EVP_aes_128_cbc(), &ek, &eklen, iv, &public_key, 1)) return false; - - // now assign memory to out accounting for data, and cipher block size, key length, and key length val - out = new unsigned char[inlen + cipher_block_size + size_net_ekl + eklen + EVP_MAX_IV_LENGTH]; - - net_ekl = htonl(eklen); - memcpy((unsigned char*)out + out_offset, &net_ekl, size_net_ekl); - out_offset += size_net_ekl; - - memcpy((unsigned char*)out + out_offset, ek, eklen); - out_offset += eklen; - - memcpy((unsigned char*)out + out_offset, iv, EVP_MAX_IV_LENGTH); - out_offset += EVP_MAX_IV_LENGTH; - - // now encrypt actual data - if(!EVP_SealUpdate(&ctx, (unsigned char*) out + out_offset, &out_currOffset, (unsigned char*) in, inlen)) return false; - - // move along to partial block space - out_offset += out_currOffset; - - // add padding - if(!EVP_SealFinal(&ctx, (unsigned char*) out + out_offset, &out_currOffset)) return false; - - // move to end - out_offset += out_currOffset; - - // make sure offset has not gone passed valid memory bounds - if(out_offset > max_outlen) return false; - - // free encrypted key data - free(ek); - - outlen = out_offset; - return true; - - delete[] ek; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3DistribSecurity::encrypt() finished with outlen : " << outlen << std::endl; -#endif - - return true; -} - - -bool p3DistribSecurity::decrypt(void *& out, int & outlen, const void *in, int inlen, EVP_PKEY *privateKey) -{ - -#ifdef DISTRIB_DEBUG - std::cerr << "p3DistribSecurity::decrypt() " << std::endl; -#endif - - EVP_CIPHER_CTX ctx; - int eklen = 0, net_ekl = 0; - unsigned char *ek = NULL; - unsigned char iv[EVP_MAX_IV_LENGTH]; - ek = (unsigned char*)malloc(EVP_PKEY_size(privateKey)); - EVP_CIPHER_CTX_init(&ctx); - - int in_offset = 0, out_currOffset = 0; - int size_net_ekl = sizeof(net_ekl); - - memcpy(&net_ekl, (unsigned char*)in, size_net_ekl); - eklen = ntohl(net_ekl); - in_offset += size_net_ekl; - - memcpy(ek, (unsigned char*)in + in_offset, eklen); - in_offset += eklen; - - memcpy(iv, (unsigned char*)in + in_offset, EVP_MAX_IV_LENGTH); - in_offset += EVP_MAX_IV_LENGTH; - - const EVP_CIPHER* cipher = EVP_aes_128_cbc(); - - if(!EVP_OpenInit(&ctx, cipher, ek, eklen, iv, privateKey)) return false; - - out = new unsigned char[inlen - in_offset]; - - if(!EVP_OpenUpdate(&ctx, (unsigned char*) out, &out_currOffset, (unsigned char*)in + in_offset, inlen - in_offset)) return false; - - in_offset += out_currOffset; - outlen += out_currOffset; - - if(!EVP_OpenFinal(&ctx, (unsigned char*)out + out_currOffset, &out_currOffset)) return false; - - outlen += out_currOffset; - - free(ek); - - return true; -} - -std::string p3DistribSecurity::getRsaKeySign(RSA *pubkey) -{ - int len = BN_num_bytes(pubkey -> n); - unsigned char tmp[len]; - BN_bn2bin(pubkey -> n, tmp); - - // copy first CERTSIGNLEN bytes... - if (len > CERTSIGNLEN) - { - len = CERTSIGNLEN; - } - - std::string id; - for(uint32_t i = 0; i < CERTSIGNLEN; i++) - { - rs_sprintf_append(id, "%02x", (uint16_t) (((uint8_t *) (tmp))[i])); - } - - return id; -} - - -bool p3DistribSecurity::validateDistribGrp(RsDistribGrp *newGrp) -{ -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::validateDistribGrp()"; - std::cerr << std::endl; -#endif - - /* check signature */ - RsSerialType *serialType = new RsDistribSerialiser(); - - - - /* copy out signature (shallow copy) */ - RsTlvKeySignature tmpSign = newGrp->adminSignature; - unsigned char *sigbuf = (unsigned char *) tmpSign.signData.bin_data; - unsigned int siglen = tmpSign.signData.bin_len; - - /* clear signature */ - newGrp->adminSignature.TlvClear(); - - uint32_t size = serialType->size(newGrp); - char* data = new char[size]; - - serialType->serialise(newGrp, data, &size); - - - const unsigned char *keyptr = (const unsigned char *) newGrp->adminKey.keyData.bin_data; - long keylen = newGrp->adminKey.keyData.bin_len; - - /* extract admin key */ - RSA *rsakey = d2i_RSAPublicKey(NULL, &(keyptr), keylen); - - EVP_PKEY *key = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(key, rsakey); - - /* calc and check signature */ - EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); - - EVP_VerifyInit(mdctx, EVP_sha1()); - EVP_VerifyUpdate(mdctx, data, size); - int ans = EVP_VerifyFinal(mdctx, sigbuf, siglen, key); - - - /* restore signature */ - newGrp->adminSignature = tmpSign; - tmpSign.TlvClear(); - - /* clean up */ - EVP_PKEY_free(key); - delete serialType; - EVP_MD_CTX_destroy(mdctx); - delete[] data; - - if (ans == 1) - return true; - -#ifdef DISTRIB_DEBUG - std::cerr << "p3GroupDistrib::validateDistribGrp() Signature invalid"; - std::cerr << std::endl; -#endif - return false; - -} - - - -void p3DistribSecurity::setRSAPublicKey(RsTlvSecurityKey & key, RSA *rsa_pub) -{ - unsigned char data[10240]; /* more than enough space */ - unsigned char *ptr = data; - int reqspace = i2d_RSAPublicKey(rsa_pub, &ptr); - - key.keyData.setBinData(data, reqspace); - - std::string keyId = getRsaKeySign(rsa_pub); - key.keyId = keyId; -} - - - -void p3DistribSecurity::setRSAPrivateKey(RsTlvSecurityKey & key, RSA *rsa_priv) -{ - unsigned char data[10240]; /* more than enough space */ - unsigned char *ptr = data; - int reqspace = i2d_RSAPrivateKey(rsa_priv, &ptr); - - key.keyData.setBinData(data, reqspace); - - std::string keyId = getRsaKeySign(rsa_priv); - key.keyId = keyId; -} - - - -RSA *p3DistribSecurity::extractPrivateKey(RsTlvSecurityKey & key) -{ - const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data; - long keylen = key.keyData.bin_len; - - /* extract admin key */ - RSA *rsakey = d2i_RSAPrivateKey(NULL, &(keyptr), keylen); - - return rsakey; -} - - diff --git a/libretroshare/src/distrib/p3distribsecurity.h b/libretroshare/src/distrib/p3distribsecurity.h deleted file mode 100644 index 44a1919d2..000000000 --- a/libretroshare/src/distrib/p3distribsecurity.h +++ /dev/null @@ -1,132 +0,0 @@ -/* - * libretroshare/src/distrib: p3distribverify.h - * - * 3P/PQI network interface for RetroShare. - * - * Copyright 2008-2010 by Robert Fernie - * 2011 Christopher Evi-Parker - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef P3DISTRIBVERIFY_H_ -#define P3DISTRIBVERIFY_H_ - -#include "serialiser/rstlvkeys.h" -#include "distrib/p3distrib.h" - -#include -#include - - -/*! - * This contains functionality for performing security - * operations needed to validate data received in p3GroupDistrib - * Also has functionality to receive data - */ -class p3DistribSecurity { - -public: - - p3DistribSecurity(); - ~p3DistribSecurity(); - - /*! - * extracts the public key from an RsTlvSecurityKey - * @param key RsTlvSecurityKey to extract public RSA key from - * @return pointer to the public RSA key if successful, null otherwise - */ - static RSA *extractPublicKey(RsTlvSecurityKey &key); - - /*! - * extracts the public key from an RsTlvSecurityKey - * @param key RsTlvSecurityKey to extract private RSA key from - * @return pointer to the private RSA key if successful, null otherwise - */ - static RSA *extractPrivateKey(RsTlvSecurityKey &key); - - /*! - * stores the rsa public key in a RsTlvSecurityKey - * @param key RsTlvSecurityKey to store the public rsa key in - * @param rsa_pub - */ - static void setRSAPublicKey(RsTlvSecurityKey &key, RSA *rsa_pub); - - /*! - * stores the rsa private key in a RsTlvSecurityKey - * @param key stores the rsa private key in a RsTlvSecurityKey - * @param rsa_priv the rsa private key to store - */ - static void setRSAPrivateKey(RsTlvSecurityKey &key, RSA *rsa_priv); - - /*! - * extracts signature from RSA key - * @param pubkey - * @return signature of RSA key in hex format - */ - static std::string getRsaKeySign(RSA *pubkey); - - /*! - * extracts the signature and stores it in a string - * in hex format - * @param data - * @param len - * @return - */ - static std::string getBinDataSign(void *data, int len); - - /*! - * Encrypts data using envelope encryption (taken from open ssl's evp_sealinit ) - * only full publish key holders can encrypt data for given group - *@param out - *@param outlen - *@param in - *@param inlen - */ - static bool encrypt(void *&out, int &outlen, const void *in, int inlen, EVP_PKEY *privateKey); - - - /** - * Decrypts data using evelope decryption (taken from open ssl's evp_sealinit ) - * only full publish key holders can decrypt data for a group - * @param out where decrypted data is written to - * @param outlen - * @param in - * @param inlen - * @return false if encryption failed - */ - static bool decrypt(void *&out, int &outlen, const void *in, int inlen, EVP_PKEY *privateKey); - - /*! - * uses grp signature to check if group has been - * tampered with - * @param newGrp - * @return true if group valid false otherwise - */ - static bool validateDistribGrp(RsDistribGrp *newGrp); - - /*! - * uses groupinfo public key to verify signature of signed message - * @param info groupinfo for which msg is meant for - * @param msg - * @return false if verfication of signature is not passed - */ - static bool validateDistribSignedMsg(GroupInfo &info, RsDistribSignedMsg *msg); -}; - -#endif /* P3DISTRIBVERIFY_H_ */ diff --git a/libretroshare/src/ft/ftdata.h b/libretroshare/src/ft/ftdata.h index a60851498..ae84a1811 100644 --- a/libretroshare/src/ft/ftdata.h +++ b/libretroshare/src/ft/ftdata.h @@ -64,10 +64,6 @@ class ftDataSend virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) = 0; virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) = 0; - /// Send a request for a chunk crc map - virtual bool sendCRC32MapRequest(const std::string& peer_id,const std::string& hash) = 0; - /// Send a chunk crc map - virtual bool sendCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) = 0; /// Send a request for a chunk crc map virtual bool sendSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_number) = 0; /// Send a chunk crc map @@ -94,11 +90,6 @@ class ftDataRecv /// Send a chunk map virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) = 0; - /// Send a request for a chunk map - virtual bool recvCRC32MapRequest(const std::string& peer_id,const std::string& hash) = 0; - - /// Send a chunk map - virtual bool recvCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crcmap) = 0; virtual bool recvSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_id) = 0; virtual bool recvSingleChunkCRC(const std::string& peer_id,const std::string& hash,uint32_t chunk_id,const Sha1CheckSum& sum) = 0; diff --git a/libretroshare/src/ft/ftdatamultiplex.cc b/libretroshare/src/ft/ftdatamultiplex.cc index 6267a05b2..c7480d181 100644 --- a/libretroshare/src/ft/ftdatamultiplex.cc +++ b/libretroshare/src/ft/ftdatamultiplex.cc @@ -60,7 +60,7 @@ const uint32_t FT_DATA = 0x0001; // data cuhnk to be stored const uint32_t FT_DATA_REQ = 0x0002; // data request to be treated const uint32_t FT_CLIENT_CHUNK_MAP_REQ = 0x0003; // chunk map request to be treated by client const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map request to be treated by server -const uint32_t FT_CRC32MAP_REQ = 0x0005; // crc32 map request to be treated by server +//const uint32_t FT_CRC32MAP_REQ = 0x0005; // crc32 map request to be treated by server const uint32_t FT_CLIENT_CHUNK_CRC_REQ = 0x0006; // chunk sha1 crc request to be treated ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data) @@ -273,19 +273,6 @@ bool ftDataMultiplex::recvChunkMapRequest(const std::string& peerId, const std:: return true; } -bool ftDataMultiplex::recvCRC32MapRequest(const std::string& peerId, const std::string& hash) -{ -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::recvChunkMapRequest() Server Recv"; - std::cerr << std::endl; -#endif - /* Store in Queue */ - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - - mRequestQueue.push_back(ftRequest(FT_CRC32MAP_REQ,peerId,hash,0,0,0,NULL)); - - return true; -} bool ftDataMultiplex::recvSingleChunkCRCRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_number) { #ifdef MPLEX_DEBUG @@ -299,27 +286,6 @@ bool ftDataMultiplex::recvSingleChunkCRCRequest(const std::string& peerId, const return true; } -class CRC32Thread: public RsThread -{ - public: - CRC32Thread(ftDataMultiplex *dataplex,const std::string& peerId,const std::string& hash) - : _plex(dataplex),_finished(false),_peerId(peerId),_hash(hash) {} - - virtual void run() - { -#ifdef MPLEX_DEBUG - std::cerr << "CRC32Thread is running for file " << _hash << std::endl; -#endif - _plex->computeAndSendCRC32Map(_peerId,_hash) ; - _finished = true ; - } - bool finished() { return _finished ; } - private: - ftDataMultiplex *_plex ; - bool _finished ; - std::string _peerId ; - std::string _hash ; -}; /*********** BACKGROUND THREAD OPERATIONS ***********/ bool ftDataMultiplex::workQueued() @@ -341,7 +307,6 @@ bool ftDataMultiplex::workQueued() bool ftDataMultiplex::doWork() { bool doRequests = true; - time_t now = time(NULL) ; /* Handle All the current Requests */ while(doRequests) @@ -396,14 +361,6 @@ bool ftDataMultiplex::doWork() handleRecvServerChunkMapRequest(req.mPeerId,req.mHash) ; break ; - case FT_CRC32MAP_REQ: -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CRC32_MAP_REQ"; - std::cerr << std::endl; -#endif - handleRecvCRC32MapRequest(req.mPeerId,req.mHash) ; - break ; - case FT_CLIENT_CHUNK_CRC_REQ: #ifdef MPLEX_DEBUG std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CHUNK_CRC_REQ"; @@ -421,47 +378,6 @@ bool ftDataMultiplex::doWork() } } - // Look for potentially finished CRC32Map threads, and destroys them. - - { - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - - for(std::list::iterator lit(_crc32map_threads.begin());lit!=_crc32map_threads.end();) - if((*lit)->finished()) - { - std::cerr << "ftDataMultiplex::doWork: thread " << *lit << " ended. Deleting it." << std::endl; - (*lit)->join() ; - delete (*lit) ; - std::list::iterator tmp(lit) ; - ++lit ; - _crc32map_threads.erase(tmp) ; - } - else - { - std::cerr << "ftDataMultiplex::doWork: thread " << *lit << " still working. Not quitting it." << std::endl; - ++lit ; - } - - // Take the opportunity to cleanup the list, so that it cannot grow indefinitely -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::doWork: Cleaning up list of cached maps." << std::endl ; -#endif - - // Keep CRC32 maps in cache for 30 mins max. - // - for(std::map >::iterator it = _cached_crc32maps.begin();it!=_cached_crc32maps.end();) - if(it->second.first + 30*60 < now) - { - std::cerr << "Removing cached map for file " << it->first << " that was kept for too long now." << std::endl; - - std::map >::iterator tmp(it) ; - ++it ; - _cached_crc32maps.erase(tmp) ; - } - else - ++it ; - } - /* Only Handle One Search Per Period.... * Lower Priority */ @@ -583,27 +499,6 @@ bool ftDataMultiplex::dispatchReceivedChunkCheckSum() return true ; } -bool ftDataMultiplex::recvCRC32Map(const std::string& /*peerId*/, const std::string& hash,const CRC32Map& crc_map) -{ - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - - std::map::iterator it = mClients.find(hash); - - if(it == mClients.end()) - { - std::cerr << "ftDataMultiplex::recvCRCMap() ERROR: No matching Client for CRC32map. This is an error. " << hash << " !" << std::endl; - /* error */ - return false; - } - -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::recvCRCMap() Passing crc map of file " << hash << ", to FT Module" << std::endl; -#endif - - (it->second).mModule->addCRC32Map(crc_map); - return true ; -} - // A chunk map has arrived. It can be two different situations: // - an uploader has sent his chunk map, so we need to store it in the corresponding ftFileProvider // - a source for a download has sent his chunk map, so we need to send it to the corresponding ftFileCreator. @@ -654,152 +549,6 @@ bool ftDataMultiplex::recvChunkMap(const std::string& peerId, const std::string& return false; } - -bool ftDataMultiplex::handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash) -{ - bool found = false ; - CRC32Map cmap ; - - // 1 - look into cache - -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() : source " << peerId << " asked for CRC32 map for file " << hash << std::endl; -#endif - { - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - std::map >::iterator it = _cached_crc32maps.find(hash) ; - - if(it != _cached_crc32maps.end()) - { - cmap = it->second.second ; - it->second.first = time(NULL) ; // update time stamp - found = true ; -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() : CRC32 map found in cache !!" << std::endl; -#endif - - } - } - - if(found) - { -#ifdef MPLEX_DEBUG - std::cerr << "File CRC32 map was obtained successfully. Sending it." << std::endl ; -#endif - - mDataSend->sendCRC32Map(peerId,hash,cmap); - return true ; - } - else - { - std::cerr << "File CRC32 Not found. Computing it." << std::endl ; - - { - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - if(_crc32map_threads.size() > 1) - { - std::cerr << "Too many threads already computing CRC32Maps (2 is the current maximum)! Giving up." << std::endl; - return false ; - } - } - - CRC32Thread *thread = new CRC32Thread(this,peerId,hash); - - { - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - _crc32map_threads.push_back(thread) ; - } - thread->start() ; - return true ; - } -} - -bool ftDataMultiplex::computeAndSendCRC32Map(const std::string& peerId, const std::string& hash) -{ - bool found ; - std::map::iterator it ; - std::string filename ; - uint64_t filesize =0; - - // 1 - look into the list of servers - { - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - - it = mServers.find(hash) ; - - if(it == mServers.end()) - found = false ; - } - - // 2 - if not found, create a server. - // - if(!found) - { -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() ERROR: No matching file Provider for hash " << hash ; - std::cerr << std::endl; -#endif - if(!handleSearchRequest(peerId,hash)) - return false ; - -#ifdef MPLEX_DEBUG - std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() A new file Provider has been made up for hash " << hash ; - std::cerr << std::endl; -#endif - } - - { - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - it = mServers.find(hash) ; - - if(it == mServers.end()) // handleSearchRequest should have filled mServers[hash], but we have been off-mutex since, - { - std::cerr << "Could definitely not find a provider for file " << hash << ". Maybe the file does not exist?" << std::endl; - return false ; // so it's safer to check again. - } - else - { - filesize = it->second->fileSize() ; - filename = it->second->fileName() ; - } - } - -#ifdef MPLEX_DEBUG - std::cerr << "Computing CRC32Map for file " << filename << ", hash=" << hash << ", size=" << filesize << std::endl; -#endif - - FILE *fd = RsDirUtil::rs_fopen(filename.c_str(),"rb") ; - - if(fd == NULL) - { - std::cerr << "Could not open file " << filename << " for read!! CRC32Map computation cancelled." << std::endl ; - return false ; - } - - CRC32Map cmap ; - if(!RsDirUtil::crc32File(fd,filesize,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,cmap)) - { - std::cerr << "CRC32Map computation failed." << std::endl ; - fclose(fd) ; - return false ; - } - fclose(fd) ; - - { - RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ - std::cerr << "File CRC32 was successfully computed. Storing it into cache." << std::endl ; - - _cached_crc32maps[hash] = std::pair(time(NULL),cmap) ; - } - -#ifdef MPLEX_DEBUG - std::cerr << "File CRC32 was successfully computed. Sending it." << std::endl ; -#endif - mDataSend->sendCRC32Map(peerId,hash,cmap); - - return true ; -} - bool ftDataMultiplex::handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) { CompressedChunkMap cmap ; @@ -1152,10 +901,6 @@ bool ftDataMultiplex::sendChunkMapRequest(const std::string& peer_id,const std:: { return mDataSend->sendChunkMapRequest(peer_id,hash,is_client); } -bool ftDataMultiplex::sendCRC32MapRequest(const std::string& peer_id,const std::string& hash) -{ - return mDataSend->sendCRC32MapRequest(peer_id,hash); -} bool ftDataMultiplex::sendSingleChunkCRCRequests(const std::string& hash, const std::vector& to_ask) { RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ diff --git a/libretroshare/src/ft/ftdatamultiplex.h b/libretroshare/src/ft/ftdatamultiplex.h index 54cdae691..23bac4cf8 100644 --- a/libretroshare/src/ft/ftdatamultiplex.h +++ b/libretroshare/src/ft/ftdatamultiplex.h @@ -36,7 +36,6 @@ class ftTransferModule; class ftFileProvider; class ftFileCreator; class ftSearch; -class CRC32Thread; #include #include @@ -121,11 +120,6 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread /* Server/client Send */ bool sendChunkMapRequest(const std::string& peerId, const std::string& hash,bool is_client) ; - /* Client Send */ - bool sendCRC32MapRequest(const std::string& peerId, const std::string& hash) ; - - /* called from a separate thread */ - bool computeAndSendCRC32Map(const std::string& peerId, const std::string& hash) ; /* called from a separate thread */ bool sendSingleChunkCRCRequests(const std::string& hash, const std::vector& to_ask) ; @@ -143,10 +137,6 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread virtual bool recvChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) ; /// Receive a chunk map virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ; - /// Receive a CRC map - virtual bool recvCRC32Map(const std::string& peer_id,const std::string& hash,const CRC32Map& crc_map) ; - /// Receive a CRC map request - virtual bool recvCRC32MapRequest(const std::string& peer_id,const std::string& hash) ; virtual bool recvSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_id) ; virtual bool recvSingleChunkCRC(const std::string& peer_id,const std::string& hash,uint32_t chunk_id,const Sha1CheckSum& sum) ; @@ -170,7 +160,6 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread bool handleSearchRequest(const std::string& peerId, const std::string& hash); bool handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) ; bool handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash) ; - bool handleRecvCRC32MapRequest(const std::string& peerId, const std::string& hash) ; bool handleRecvChunkCrcRequest(const std::string& peerId, const std::string& hash,uint32_t chunk_id) ; /* We end up doing the actual server job here */ @@ -185,9 +174,6 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread std::list mSearchQueue; // std::map mUnknownHashs; - std::list _crc32map_threads ; - std::map > _cached_crc32maps ; - std::map _cached_sha1maps ; // one cache entry per file hash. Handled dynamically. ftDataSend *mDataSend; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 38aa6af66..845b1c951 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -52,23 +52,29 @@ const int ftserverzone = 29539; #include "pqi/pqi.h" #include "pqi/p3linkmgr.h" +#include "serialiser/rsfiletransferitems.h" #include "serialiser/rsserviceids.h" /*** - * #define SERVER_DEBUG 1 - * #define DEBUG_TICK 1 + * #define SERVER_DEBUG 1 + * #define SERVER_DEBUG_CACHE 1 ***/ +static const time_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priority tasks handling every 5 seconds + /* Setup */ ftServer::ftServer(p3PeerMgr *pm, p3LinkMgr *lm) - : mP3iface(NULL), mPeerMgr(pm), + : p3Service(RS_SERVICE_TYPE_FILE_TRANSFER), + mPeerMgr(pm), mLinkMgr(lm), mCacheStrapper(NULL), mFiStore(NULL), mFiMon(NULL), mFtController(NULL), mFtExtra(NULL), mFtDataplex(NULL), mFtSearch(NULL), srvMutex("ftServer") { - mCacheStrapper = new ftCacheStrapper(lm); + mCacheStrapper = new ftCacheStrapper(lm); + + addSerialType(new RsFileTransferSerialiser()) ; } void ftServer::setConfigDirectory(std::string path) @@ -88,11 +94,6 @@ void ftServer::setConfigDirectory(std::string path) RsDirUtil::checkCreateDirectory(remotecachedir) ; } -void ftServer::setP3Interface(P3Interface *pqi) -{ - mP3iface = pqi; -} - /* Control Interface */ /* add Config Items (Extra, Controller) */ @@ -182,16 +183,10 @@ void ftServer::StartupThreads() /* Dataplex */ mFtDataplex->start(); - - /* start own thread */ - start(); } void ftServer::StopThreads() { - /* stop own thread */ - join(); - /* stop Dataplex */ mFtDataplex->join(); @@ -228,31 +223,22 @@ CacheTransfer *ftServer::getCacheTransfer() return mFtController; } -void ftServer::run() + +/***************************************************************/ +/********************** RsFiles Interface **********************/ +/***************************************************************/ + +/***************************************************************/ +/********************** Controller Access **********************/ +/***************************************************************/ + +bool ftServer::ResumeTransfers() { - while(isRunning()) - { - mFtDataplex->deleteUnusedServers() ; - mFtDataplex->handlePendingCrcRequests() ; - mFtDataplex->dispatchReceivedChunkCheckSum() ; -#ifdef WIN32 - Sleep(5000); -#else - sleep(5); -#endif - } + mFtController->activate(); + + return true; } - - /***************************************************************/ - /********************** RsFiles Interface **********************/ - /***************************************************************/ - - - /***************************************************************/ - /********************** Controller Access **********************/ - /***************************************************************/ - bool ftServer::checkHash(const std::string& hash,std::string& error_string) { static const uint32_t HASH_LENGTH = 40 ; @@ -465,15 +451,11 @@ RsTurtleGenericTunnelItem *ftServer::deserialiseItem(void *data,uint32_t size) c #ifdef SERVER_DEBUG std::cerr << "p3turtle: deserialising packet: " << std::endl ; #endif -#ifdef SERVER_DEBUG if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_TURTLE != getRsItemService(rstype))) { -#ifdef SERVER_DEBUG std::cerr << " Wrong type !!" << std::endl ; -#endif return NULL; /* wrong type */ } -#endif switch(getRsItemSubType(rstype)) { @@ -857,10 +839,15 @@ bool ftServer::shareDownloadDirectory(bool share) /**************** Config Interface *****************************/ /***************************************************************/ - /* Key Functions to be overloaded for Full Configuration */ +/* Key Functions to be overloaded for Full Configuration */ RsSerialiser *ftServer::setupSerialiser() { - return NULL; + RsSerialiser *rss = new RsSerialiser ; + rss->addSerialType(new RsFileTransferSerialiser) ; + + //rss->addSerialType(new RsGeneralConfigSerialiser()); + + return rss ; } bool ftServer::saveList(bool &/*cleanup*/, std::list& /*list*/) @@ -887,6 +874,9 @@ bool ftServer::loadConfigMap(std::map &/*configMap*/) /* Client Send */ bool ftServer::sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize) { +#ifdef SERVER_DEBUG + std::cerr << "ftServer::sendDataRequest() to peer " << peerId << " for hash " << hash << ", offset=" << offset << ", chunk size="<< chunksize << std::endl; +#endif if(mTurtleRouter->isTurtlePeer(peerId)) { RsTurtleFileRequestItem *item = new RsTurtleFileRequestItem ; @@ -898,9 +888,10 @@ bool ftServer::sendDataRequest(const std::string& peerId, const std::string& has } else { + std::cerr << "ftServer::sendDataRequest() " <PeerId(peerId); @@ -913,7 +904,7 @@ bool ftServer::sendDataRequest(const std::string& peerId, const std::string& has rfi->fileoffset = offset; /* ftr->offset; */ rfi->chunksize = chunksize; /* ftr->chunk; */ - mP3iface->SendFileRequest(rfi); + sendItem(rfi); } return true; @@ -921,6 +912,9 @@ bool ftServer::sendDataRequest(const std::string& peerId, const std::string& has bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string& hash,bool is_client) { +#ifdef SERVER_DEBUG + std::cerr << "ftServer::sendChunkMapRequest() to peer " << peerId << " for hash " << hash << std::endl; +#endif if(mTurtleRouter->isTurtlePeer(peerId)) { RsTurtleFileMapRequestItem *item = new RsTurtleFileMapRequestItem ; @@ -930,7 +924,7 @@ bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string& { /* create a packet */ /* push to networking part */ - RsFileChunkMapRequest *rfi = new RsFileChunkMapRequest(); + RsFileTransferChunkMapRequestItem *rfi = new RsFileTransferChunkMapRequestItem(); /* id */ rfi->PeerId(peerId); @@ -939,7 +933,7 @@ bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string& rfi->hash = hash; /* ftr->hash; */ rfi->is_client = is_client ; - mP3iface->SendFileChunkMapRequest(rfi); + sendItem(rfi); } return true ; @@ -947,6 +941,9 @@ bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string& bool ftServer::sendChunkMap(const std::string& peerId,const std::string& hash,const CompressedChunkMap& map,bool is_client) { +#ifdef SERVER_DEBUG + std::cerr << "ftServer::sendChunkMap() to peer " << peerId << " for hash " << hash << std::endl; +#endif if(mTurtleRouter->isTurtlePeer(peerId)) { RsTurtleFileMapItem *item = new RsTurtleFileMapItem ; @@ -957,7 +954,7 @@ bool ftServer::sendChunkMap(const std::string& peerId,const std::string& hash,co { /* create a packet */ /* push to networking part */ - RsFileChunkMap *rfi = new RsFileChunkMap(); + RsFileTransferChunkMapItem *rfi = new RsFileTransferChunkMapItem(); /* id */ rfi->PeerId(peerId); @@ -967,38 +964,17 @@ bool ftServer::sendChunkMap(const std::string& peerId,const std::string& hash,co rfi->is_client = is_client; /* ftr->hash; */ rfi->compressed_map = map; /* ftr->hash; */ - mP3iface->SendFileChunkMap(rfi); + sendItem(rfi); } return true ; } -bool ftServer::sendCRC32MapRequest(const std::string& peerId,const std::string& hash) -{ - if(mTurtleRouter->isTurtlePeer(peerId)) - { - RsTurtleFileCrcRequestItem *item = new RsTurtleFileCrcRequestItem; - mTurtleRouter->sendTurtleData(peerId,item) ; - } - else - { - /* create a packet */ - /* push to networking part */ - RsFileCRC32MapRequest *rfi = new RsFileCRC32MapRequest(); - - /* id */ - rfi->PeerId(peerId); - - /* file info */ - rfi->hash = hash; /* ftr->hash; */ - - mP3iface->SendFileCRC32MapRequest(rfi); - } - - return true ; -} bool ftServer::sendSingleChunkCRCRequest(const std::string& peerId,const std::string& hash,uint32_t chunk_number) { +#ifdef SERVER_DEBUG + std::cerr << "ftServer::sendSingleCRCRequest() to peer " << peerId << " for hash " << hash << ", chunk number=" << chunk_number << std::endl; +#endif if(mTurtleRouter->isTurtlePeer(peerId)) { RsTurtleChunkCrcRequestItem *item = new RsTurtleChunkCrcRequestItem; @@ -1010,7 +986,7 @@ bool ftServer::sendSingleChunkCRCRequest(const std::string& peerId,const std::st { /* create a packet */ /* push to networking part */ - RsFileSingleChunkCrcRequest *rfi = new RsFileSingleChunkCrcRequest(); + RsFileTransferSingleChunkCrcRequestItem *rfi = new RsFileTransferSingleChunkCrcRequestItem(); /* id */ rfi->PeerId(peerId); @@ -1019,41 +995,17 @@ bool ftServer::sendSingleChunkCRCRequest(const std::string& peerId,const std::st rfi->hash = hash; /* ftr->hash; */ rfi->chunk_number = chunk_number ; - mP3iface->SendFileSingleChunkCrcRequest(rfi); + sendItem(rfi); } return true ; } -bool ftServer::sendCRC32Map(const std::string& peerId,const std::string& hash,const CRC32Map& crcmap) -{ - if(mTurtleRouter->isTurtlePeer(peerId)) - { - RsTurtleFileCrcItem *item = new RsTurtleFileCrcItem ; - item->crc_map = crcmap ; - - mTurtleRouter->sendTurtleData(peerId,item) ; - } - else - { - /* create a packet */ - /* push to networking part */ - RsFileCRC32Map *rfi = new RsFileCRC32Map(); - - /* id */ - rfi->PeerId(peerId); - - /* file info */ - rfi->hash = hash; /* ftr->hash; */ - rfi->crc_map = crcmap; /* ftr->hash; */ - - mP3iface->SendFileCRC32Map(rfi); - } - - return true ; -} bool ftServer::sendSingleChunkCRC(const std::string& peerId,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc) { +#ifdef SERVER_DEBUG + std::cerr << "ftServer::sendSingleCRC() to peer " << peerId << " for hash " << hash << ", chunk number=" << chunk_number << std::endl; +#endif if(mTurtleRouter->isTurtlePeer(peerId)) { RsTurtleChunkCrcItem *item = new RsTurtleChunkCrcItem; @@ -1066,7 +1018,7 @@ bool ftServer::sendSingleChunkCRC(const std::string& peerId,const std::string& h { /* create a packet */ /* push to networking part */ - RsFileSingleChunkCrc *rfi = new RsFileSingleChunkCrc(); + RsFileTransferSingleChunkCrcItem *rfi = new RsFileTransferSingleChunkCrcItem(); /* id */ rfi->PeerId(peerId); @@ -1076,16 +1028,12 @@ bool ftServer::sendSingleChunkCRC(const std::string& peerId,const std::string& h rfi->check_sum = crc; rfi->chunk_number = chunk_number; - mP3iface->SendFileSingleChunkCrc(rfi); + sendItem(rfi); } return true ; } -//const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */ -//const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */ -const uint32_t MAX_FT_CHUNK = 8 * 1024; /* 16K */ - /* Server Send */ bool ftServer::sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data) { @@ -1106,6 +1054,11 @@ bool ftServer::sendData(const std::string& peerId, const std::string& hash, uint while(tosend > 0) { + //static const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */ + //static const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */ + // + static const uint32_t MAX_FT_CHUNK = 8 * 1024; /* 16K */ + /* workout size */ chunk = MAX_FT_CHUNK; if (chunk > tosend) @@ -1135,7 +1088,7 @@ bool ftServer::sendData(const std::string& peerId, const std::string& hash, uint } else { - RsFileData *rfd = new RsFileData(); + RsFileTransferDataItem *rfd = new RsFileTransferDataItem(); /* set id */ rfd->PeerId(peerId); @@ -1153,7 +1106,7 @@ bool ftServer::sendData(const std::string& peerId, const std::string& hash, uint /* file data */ rfd->fd.binData.setBinData( &(((uint8_t *) data)[offset]), chunk); - mP3iface->SendFileData(rfd); + sendItem(rfd); /* print the data pointer */ #ifdef SERVER_DEBUG @@ -1176,6 +1129,8 @@ bool ftServer::sendData(const std::string& peerId, const std::string& hash, uint return true; } +// Dont delete the item. The client (p3turtle) is doing it after calling this. +// void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, const std::string& hash, const std::string& virtual_peer_id, @@ -1186,6 +1141,9 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_FILE_REQUEST: { RsTurtleFileRequestItem *item = dynamic_cast(i) ; +#ifdef SERVER_DEBUG + std::cerr << "ftServer::receiveTurtleData(): received file data request for " << hash << " from peer " << virtual_peer_id << std::endl; +#endif getMultiplexer()->recvDataRequest(virtual_peer_id,hash,0,item->chunk_offset,item->chunk_size) ; } break ; @@ -1193,6 +1151,9 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_FILE_DATA : { RsTurtleFileDataItem *item = dynamic_cast(i) ; +#ifdef SERVER_DEBUG + std::cerr << "ftServer::receiveTurtleData(): received file data for " << hash << " from peer " << virtual_peer_id << std::endl; +#endif getMultiplexer()->recvData(virtual_peer_id,hash,0,item->chunk_offset,item->chunk_size,item->chunk_data) ; item->chunk_data = NULL ; // this prevents deletion in the destructor of RsFileDataItem, because data will be deleted @@ -1204,33 +1165,29 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_FILE_MAP : { RsTurtleFileMapItem *item = dynamic_cast(i) ; +#ifdef SERVER_DEBUG + std::cerr << "ftServer::receiveTurtleData(): received chunk map for hash " << hash << " from peer " << virtual_peer_id << std::endl; +#endif getMultiplexer()->recvChunkMap(virtual_peer_id,hash,item->compressed_map,direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ; } break ; case RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST: { - RsTurtleFileMapRequestItem *item = dynamic_cast(i) ; + //RsTurtleFileMapRequestItem *item = dynamic_cast(i) ; +#ifdef SERVER_DEBUG + std::cerr << "ftServer::receiveTurtleData(): received chunkmap request for hash " << hash << " from peer " << virtual_peer_id << std::endl; +#endif getMultiplexer()->recvChunkMapRequest(virtual_peer_id,hash,direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ; } break ; - case RS_TURTLE_SUBTYPE_FILE_CRC : - { - RsTurtleFileCrcItem *item = dynamic_cast(i) ; - getMultiplexer()->recvCRC32Map(virtual_peer_id,hash,item->crc_map) ; - } - break ; - - case RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST: - { - getMultiplexer()->recvCRC32MapRequest(virtual_peer_id,hash) ; - } - break ; - case RS_TURTLE_SUBTYPE_CHUNK_CRC : { RsTurtleChunkCrcItem *item = dynamic_cast(i) ; +#ifdef SERVER_DEBUG + std::cerr << "ftServer::receiveTurtleData(): received single chunk CRC for hash " << hash << " from peer " << virtual_peer_id << std::endl; +#endif getMultiplexer()->recvSingleChunkCRC(virtual_peer_id,hash,item->chunk_number,item->check_sum) ; } break ; @@ -1238,6 +1195,9 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST: { RsTurtleChunkCrcRequestItem *item = dynamic_cast(i) ; +#ifdef SERVER_DEBUG + std::cerr << "ftServer::receiveTurtleData(): received single chunk CRC request for hash " << hash << " from peer " << virtual_peer_id << std::endl; +#endif getMultiplexer()->recvSingleChunkCRCRequest(virtual_peer_id,hash,item->chunk_number) ; } break ; @@ -1246,358 +1206,189 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, } } - /* NB: The rsCore lock must be activated before calling this. * This Lock should be moved lower into the system... * most likely destination is in ftServer. */ int ftServer::tick() { - rslog(RSL_DEBUG_BASIC, ftserverzone, - "filedexserver::tick()"); + bool moreToTick = false ; - if (mP3iface == NULL) + if(handleIncoming()) + moreToTick = true; + + if(handleCacheData()) + moreToTick = true; + + static time_t last_law_priority_tasks_handling_time = 0 ; + time_t now = time(NULL) ; + + if(last_law_priority_tasks_handling_time + FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD < now) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::tick() ERROR: mP3iface == NULL"; -#endif + last_law_priority_tasks_handling_time = now ; - rslog(RSL_DEBUG_BASIC, ftserverzone, - "filedexserver::tick() Invalid Interface()"); - - return 1; + mFtDataplex->deleteUnusedServers() ; + mFtDataplex->handlePendingCrcRequests() ; + mFtDataplex->dispatchReceivedChunkCheckSum() ; } - int moreToTick = 0; - - if (0 < mP3iface -> tick()) - { - moreToTick = 1; -#ifdef DEBUG_TICK - std::cerr << "filedexserver::tick() moreToTick from mP3iface" << std::endl; -#endif - } - - if (0 < handleInputQueues()) - { - moreToTick = 1; -#ifdef DEBUG_TICK - std::cerr << "filedexserver::tick() moreToTick from InputQueues" << std::endl; -#endif - } - return moreToTick; -} - - -// This function needs to be divided up. -bool ftServer::handleInputQueues() -{ - bool moreToTick = false; - - if (handleCacheData()) - moreToTick = true; - - if (handleFileData()) - moreToTick = true; - return moreToTick; } -bool ftServer::handleCacheData() +bool ftServer::handleCacheData() { - // get all the incoming results.. and print to the screen. - RsCacheRequest *cr; - RsCacheItem *ci; + std::list > cacheUpdates; + std::list >::iterator it; + int i=0 ; - // Loop through Search Results. - int i = 0; - int i_init = 0; - -#ifdef SERVER_DEBUG - //std::cerr << "ftServer::handleCacheData()" << std::endl; +#ifdef SERVER_DEBUG_CACHE + std::cerr << "handleCacheData() " << std::endl; #endif - while((ci = mP3iface -> GetSearchResult()) != NULL) + mCacheStrapper->getCacheUpdates(cacheUpdates); + for(it = cacheUpdates.begin(); it != cacheUpdates.end(); it++) { + /* construct reply */ + RsFileTransferCacheItem *ci = new RsFileTransferCacheItem(); -#ifdef SERVER_DEBUG - std::cerr << "ftServer::handleCacheData() Recvd SearchResult (CacheResponse!)" << std::endl; - std::string out; - if (i++ == i_init) - { - out += "Recieved Search Results:\n"; - } - ci -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + /* id from incoming */ + ci -> PeerId(it->first); + + ci -> file.hash = (it->second).hash; + ci -> file.name = (it->second).name; + ci -> file.path = ""; // (it->second).path; + ci -> file.filesize = (it->second).size; + ci -> cacheType = (it->second).cid.type; + ci -> cacheSubId = (it->second).cid.subid; + +#ifdef SERVER_DEBUG_CACHE + std::string out2 = "Outgoing CacheStrapper Update -> RsCacheItem:\n"; + ci -> print_string(out2); + std::cerr << out2 << std::endl; #endif - /* these go to the CacheStrapper! */ - RsCacheData data; - data.pid = ci->PeerId(); - data.cid = CacheId(ci->cacheType, ci->cacheSubId); - data.path = ci->file.path; - data.name = ci->file.name; - data.hash = ci->file.hash; - data.size = ci->file.filesize; - data.recvd = time(NULL) ; + sendItem(ci); - mCacheStrapper->recvCacheResponse(data, time(NULL)); - - delete ci; + i++; } - // now requested Searches. - i_init = i; - while((cr = mP3iface -> RequestedSearch()) != NULL) - { -#ifdef SERVER_DEBUG - /* just delete these */ - std::string out = "Requested Search:\n"; - cr -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); -#endif - delete cr; - } - - - // Now handle it replacement (pushed cache results) - { - std::list > cacheUpdates; - std::list >::iterator it; - - mCacheStrapper->getCacheUpdates(cacheUpdates); - for(it = cacheUpdates.begin(); it != cacheUpdates.end(); it++) - { - /* construct reply */ - RsCacheItem *ci = new RsCacheItem(); - - /* id from incoming */ - ci -> PeerId(it->first); - - ci -> file.hash = (it->second).hash; - ci -> file.name = (it->second).name; - ci -> file.path = ""; // (it->second).path; - ci -> file.filesize = (it->second).size; - ci -> cacheType = (it->second).cid.type; - ci -> cacheSubId = (it->second).cid.subid; - -#ifdef SERVER_DEBUG - std::string out2 = "Outgoing CacheStrapper Update -> RsCacheItem:\n"; - ci -> print_string(out2); - std::cerr << out2 << std::endl; -#endif - - //rslog(RSL_DEBUG_BASIC, ftserverzone, out2.str()); - mP3iface -> SendSearchResult(ci); - - i++; - } - } - return (i > 0); + return i>0 ; } - -bool ftServer::handleFileData() +int ftServer::handleIncoming() { // now File Input. - RsFileRequest *fr; - RsFileData *fd; - RsFileChunkMapRequest *fcmr; - RsFileChunkMap *fcm; - RsFileCRC32MapRequest *fccrcmr; - RsFileCRC32Map *fccrcm; - RsFileSingleChunkCrcRequest *fscrcr; - RsFileSingleChunkCrc *fscrc; + int nhandled = 0 ; - int i_init = 0; - int i = 0; - - i_init = i; - while((fr = mP3iface -> GetFileRequest()) != NULL ) - { + RsItem *item = NULL ; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ftFiler Request" << std::endl; - std::string out; - if (i == i_init) - { - out += "Incoming(Net) File Item:\n"; - } - fr -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + std::cerr << "ftServer::handleIncoming() " << std::endl; #endif - i++; /* count */ - mFtDataplex->recvDataRequest(fr->PeerId(), - fr->file.hash, fr->file.filesize, - fr->fileoffset, fr->chunksize); - -FileInfo(ffr); - delete fr; - } - - // now File Data. - i_init = i; - while((fd = mP3iface -> GetFileData()) != NULL ) + while(NULL != (item = recvItem())) { -#ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ftFiler Data" << std::endl; - std::cerr << "hash: " << fd->fd.file.hash; - std::cerr << " length: " << fd->fd.binData.bin_len; - std::cerr << " data: " << fd->fd.binData.bin_data; - std::cerr << std::endl; + nhandled++ ; - std::string out; - if (i == i_init) + switch(item->PacketSubType()) { - out += "Incoming(Net) File Data:\n"; - } - fd -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); -#endif - i++; /* count */ - - /* incoming data */ - mFtDataplex->recvData(fd->PeerId(), - fd->fd.file.hash, fd->fd.file.filesize, - fd->fd.file_offset, - fd->fd.binData.bin_len, - fd->fd.binData.bin_data); - - /* we've stolen the data part -> so blank before delete - */ - fd->fd.binData.TlvShallowClear(); - delete fd; - } - // now file chunkmap requests - i_init = i; - while((fcmr = mP3iface -> GetFileChunkMapRequest()) != NULL ) - { + case RS_PKT_SUBTYPE_FT_DATA_REQUEST: + { + RsFileTransferDataRequestItem *f = dynamic_cast(item) ; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl; - std::string out; - if (i == i_init) - { - out += "Incoming(Net) File Data:\n"; - } - fcmr -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + std::cerr << "ftServer::handleIncoming: received data request for hash " << f->file.hash << ", offset=" << f->fileoffset << ", chunk size=" << f->chunksize << std::endl; #endif - i++; /* count */ + mFtDataplex->recvDataRequest(f->PeerId(), f->file.hash, f->file.filesize, f->fileoffset, f->chunksize); + } + break ; - /* incoming data */ - mFtDataplex->recvChunkMapRequest(fcmr->PeerId(), fcmr->hash,fcmr->is_client) ; - - delete fcmr; - } - // now file chunkmaps - i_init = i; - while((fcm = mP3iface -> GetFileChunkMap()) != NULL ) - { + case RS_PKT_SUBTYPE_FT_DATA: + { + RsFileTransferDataItem *f = dynamic_cast(item) ; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl; - std::string out; - if (i == i_init) - { - out += "Incoming(Net) File Data:\n"; - } - fcm -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + std::cerr << "ftServer::handleIncoming: received data for hash " << f->fd.file.hash << ", offset=" << f->fd.file_offset << ", chunk size=" << f->fd.binData.bin_len << std::endl; #endif - i++; /* count */ + mFtDataplex->recvData(f->PeerId(), f->fd.file.hash, f->fd.file.filesize, f->fd.file_offset, f->fd.binData.bin_len, f->fd.binData.bin_data); - /* incoming data */ - mFtDataplex->recvChunkMap(fcm->PeerId(), fcm->hash,fcm->compressed_map,fcm->is_client) ; + /* we've stolen the data part -> so blank before delete + */ + f->fd.binData.TlvShallowClear(); + } + break ; - delete fcm; - } - // now file chunkmap requests - i_init = i; - while((fccrcmr = mP3iface -> GetFileCRC32MapRequest()) != NULL ) - { + case RS_PKT_SUBTYPE_FT_CHUNK_MAP_REQUEST: + { + RsFileTransferChunkMapRequestItem *f = dynamic_cast(item) ; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl; - std::string out; - if (i == i_init) - { - out += "Incoming(Net) File Data:\n"; - } - fccrcmr -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + std::cerr << "ftServer::handleIncoming: received chunkmap request for hash " << f->hash << ", client=" << f->is_client << std::endl; #endif - i++; /* count */ + mFtDataplex->recvChunkMapRequest(f->PeerId(), f->hash,f->is_client) ; + } + break ; - /* incoming data */ - mFtDataplex->recvCRC32MapRequest(fccrcmr->PeerId(), fccrcmr->hash) ; - - delete fccrcmr; - } - // now file chunkmaps - i_init = i; - while((fccrcm = mP3iface -> GetFileCRC32Map()) != NULL ) - { + case RS_PKT_SUBTYPE_FT_CHUNK_MAP: + { + RsFileTransferChunkMapItem *f = dynamic_cast(item) ; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl; - std::string out; - if (i == i_init) - { - out += "Incoming(Net) File Data:\n"; - } - fccrcm -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + std::cerr << "ftServer::handleIncoming: received chunkmap for hash " << f->hash << ", client=" << f->is_client << /*", map=" << f->compressed_map <<*/ std::endl; #endif - i++; /* count */ + mFtDataplex->recvChunkMap(f->PeerId(), f->hash,f->compressed_map,f->is_client) ; + } + break ; - /* incoming data */ - mFtDataplex->recvCRC32Map(fccrcm->PeerId(), fccrcm->hash,fccrcm->crc_map); - - delete fccrcm; - } - // now file chunk crc requests - i_init = i; - while((fscrcr = mP3iface -> GetFileSingleChunkCrcRequest()) != NULL ) - { + case RS_PKT_SUBTYPE_FT_CHUNK_CRC_REQUEST: + { + RsFileTransferSingleChunkCrcRequestItem *f = dynamic_cast(item) ; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl; - std::string out; - if (i == i_init) - { - out += "Incoming(Net) File CRC Request:\n"; - } - fscrcr -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + std::cerr << "ftServer::handleIncoming: received single chunk crc req for hash " << f->hash << ", chunk number=" << f->chunk_number << std::endl; #endif - i++; /* count */ + mFtDataplex->recvSingleChunkCRCRequest(f->PeerId(), f->hash,f->chunk_number) ; + } + break ; - /* incoming data */ - mFtDataplex->recvSingleChunkCRCRequest(fscrcr->PeerId(), fscrcr->hash,fscrcr->chunk_number) ; - - delete fscrcr; - } - // now file chunkmaps - i_init = i; - while((fscrc = mP3iface -> GetFileSingleChunkCrc()) != NULL ) - { + case RS_PKT_SUBTYPE_FT_CHUNK_CRC: + { + RsFileTransferSingleChunkCrcItem *f = dynamic_cast(item) ; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleFileData() Recvd ChunkMap request" << std::endl; - std::string out; - if (i == i_init) - { - out += "Incoming(Net) File Data:\n"; - } - fscrc -> print_string(out); - rslog(RSL_DEBUG_BASIC, ftserverzone, out); + std::cerr << "ftServer::handleIncoming: received single chunk crc req for hash " << f->hash << ", chunk number=" << f->chunk_number << ", checksum = " << f->check_sum << std::endl; #endif - i++; /* count */ + mFtDataplex->recvSingleChunkCRC(f->PeerId(), f->hash,f->chunk_number,f->check_sum); + } + break ; - /* incoming data */ - mFtDataplex->recvSingleChunkCRC(fscrc->PeerId(), fscrc->hash,fscrc->chunk_number,fscrc->check_sum); + case RS_PKT_SUBTYPE_FT_CACHE_ITEM: + { + RsFileTransferCacheItem *ci = dynamic_cast(item) ; +#ifdef SERVER_DEBUG_CACHE + std::cerr << "ftServer::handleIncoming: received cache item hash=" << ci->file.hash << ". from peer " << ci->PeerId() << std::endl; +#endif + /* these go to the CacheStrapper! */ + RsCacheData data; + data.pid = ci->PeerId(); + data.cid = CacheId(ci->cacheType, ci->cacheSubId); + data.path = ci->file.path; + data.name = ci->file.name; + data.hash = ci->file.hash; + data.size = ci->file.filesize; + data.recvd = time(NULL) ; - delete fscrcr; + mCacheStrapper->recvCacheResponse(data, time(NULL)); + } + break ; + +// case RS_PKT_SUBTYPE_FT_CACHE_REQUEST: +// { +// // do nothing +// RsFileTransferCacheRequestItem *cr = dynamic_cast(item) ; +//#ifdef SERVER_DEBUG_CACHE +// std::cerr << "ftServer::handleIncoming: received cache request from peer " << cr->PeerId() << std::endl; +//#endif +// } +// break ; + } + + delete item ; } - if (i > 0) - { - return 1; - } - return 0; + + return nhandled; } /********************************** @@ -1617,10 +1408,3 @@ bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr) return true; } -bool ftServer::ResumeTransfers() -{ - mFtController->activate(); - - return true; -} - diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index ba3be919b..5edddb4ca 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -45,6 +45,7 @@ #include "ft/ftdata.h" #include "turtle/turtleclientservice.h" +#include "services/p3service.h" #include "retroshare/rsfiles.h" //#include "dbase/cachestrapper.h" @@ -73,7 +74,7 @@ class ftDwlQueue; class p3PeerMgr; class p3LinkMgr; -class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, public RsThread +class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService { public: @@ -87,8 +88,6 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, /* Assign important variables */ void setConfigDirectory(std::string path); - void setP3Interface(P3Interface *pqi); - /* add Config Items (Extra, Controller) */ void addConfigComponents(p3ConfigMgr *mgr); @@ -101,12 +100,6 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, void SetupFtServer() ; virtual void connectToTurtleRouter(p3turtle *p) ; - void StartupThreads(); - void StopThreads(); - - /* own thread */ - virtual void run(); - // Checks that the given hash is well formed. Used to chase // string bugs. static bool checkHash(const std::string& hash,std::string& error_string) ; @@ -125,6 +118,9 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, /************** (Implements RsFiles) ***************************/ /***************************************************************/ + void StartupThreads(); + void StopThreads(); + // member access ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; } @@ -238,8 +234,6 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, virtual bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize); virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) ; virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ; - virtual bool sendCRC32MapRequest(const std::string&, const std::string&) ; - virtual bool sendCRC32Map(const std::string&, const std::string&, const CRC32Map&) ; virtual bool sendSingleChunkCRCRequest(const std::string& peer_id,const std::string& hash,uint32_t chunk_number) ; virtual bool sendSingleChunkCRC(const std::string& peer_id,const std::string& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ; @@ -250,17 +244,12 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, bool addConfiguration(p3ConfigMgr *cfgmgr); bool ResumeTransfers(); - private: - bool handleInputQueues(); - bool handleCacheData(); - bool handleFileData(); - /******************* p3 Config Overload ************************/ protected: /* Key Functions to be overloaded for Full Configuration */ virtual RsSerialiser *setupSerialiser(); virtual bool saveList(bool &cleanup, std::list&); - virtual bool loadList(std::list& load); + virtual bool loadList(std::list& load); private: bool loadConfigMap(std::map &configMap); @@ -268,6 +257,10 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, /*************************** p3 Config Overload ********************/ + protected: + int handleIncoming() ; + bool handleCacheData() ; + private: /**** INTERNAL FUNCTIONS ***/ @@ -279,9 +272,6 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, /* no need for Mutex protection - * as each component is protected independently. */ - - P3Interface *mP3iface; /* XXX THIS NEEDS PROTECTION */ - p3PeerMgr *mPeerMgr; p3LinkMgr *mLinkMgr; @@ -295,7 +285,6 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, ftDataMultiplex *mFtDataplex; p3turtle *mTurtleRouter ; - ftFileSearch *mFtSearch; ftDwlQueue *mFtDwlQueue; @@ -304,7 +293,6 @@ class ftServer: public RsFiles, public ftDataSend, public RsTurtleClientService, std::string mConfigPath; std::string mDownloadPath; std::string mPartialsPath; - }; diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index 0517112d4..7bb95b9a5 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -530,9 +530,6 @@ int ftTransferModule::tick() case FT_TM_FLAG_CHECKING: // Check if file hash matches the hashed data checkFile() ; break ; - case FT_TM_FLAG_CHUNK_CRC: // File is waiting for CRC32 map. Check if received, and re-set matched chunks - checkCRC() ; - break ; default: break; } @@ -648,171 +645,6 @@ void ftTransferModule::forceCheck() mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING; } -bool ftTransferModule::checkCRC() -{ - // We have a finite state machine here. - // - // The states are, for each chunk, and what should be done: - // DONT_HAVE - // -> ask for the chunk CRC - // ASKED - // -> do nothing - // RECEIVED - // -> check the chunk - // CHECKED - // -> do nothing - // - // CRCs are asked by group of CRC_REQUEST_MAX_SIZE chunks at a time. The response may contain a different - // number of CRCs, depending on the availability. - // - // Server side: - // - Only complete files can compute CRCs, as the CRCs of downloaded chunks in an unchecked file are un-verified. - // - CRCs of files are cached, so that they don't get computed twice. - // -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): looking for CRC32 map." << std::endl ; -#endif - - // Loop over chunks. Collect the ones to ask for, and hash the ones received. - // If we have - // - time_t now = time(NULL) ; - - RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ - - switch(_crcmap_state) - { - case FT_TM_CRC_MAP_STATE_NOCHECK: - _crcreq_source = "" ; -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): state is NOCHECK. Doing nothing." << std::endl ; -#endif - break ; - - case FT_TM_CRC_MAP_STATE_DONT_HAVE: - { - // Check wether we have a CRC map or not. - // - std::cerr << "FT_TM_CRC_MAP_STATE_ASKED: last time is " << _crcmap_last_asked_time << std::endl ; - std::cerr << "FT_TM_CRC_MAP_STATE_ASKED: now is " << now << std::endl ; - - uint64_t threshold = (uint64_t)(FT_TM_CRC_MAP_MAX_WAIT_PER_GIG * (1+mSize/float(1024ull*1024ull*1024ull))) ; - - std::cerr << "Threshold is " << threshold << std::endl; - std::cerr << "Limit is " << (uint64_t)_crcmap_last_asked_time + threshold << std::endl ; - std::cerr << "Requested source is \"" << _crcreq_source << "\"" << std::endl; - - if( _crcreq_source != "" && (uint64_t)_crcmap_last_tunnel_keepup + 10 <= (uint64_t)now) - { -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): sending keepup to source " << _crcreq_source << std::endl ; -#endif - locked_requestData(_crcreq_source,0,(uint32_t)std::min((uint64_t)512,mSize)); - _crcmap_last_tunnel_keepup = now ; - } - - if( (uint64_t)_crcmap_last_asked_time + threshold > (uint64_t)now) - { -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): state is NOCHECK. Doing nothing." << std::endl ; -#endif - break ; - } - // Ask the ones we should ask for. We use a very coarse strategy now: we - // send the request to a random source. We'll make this more sensible - // later. - -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): state is DONT_HAVE or last request is too old. Selecting a source for asking a CRC map." << std::endl ; -#endif - if(mFileSources.empty()) - { - std::cerr << "ftTransferModule::checkCRC(): No sources available for checking file " << mHash << ": waiting 3 additional sec." <::const_iterator mit = mFileSources.begin();mit != mFileSources.end();++mit) - if(mFileCreator->sourceIsComplete(mit->first)) - { -//#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): sending CRC map request to source " << mit->first << std::endl ; -//#endif - _crcmap_last_asked_time = now ; - _crcreq_source = mit->first ; - - mMultiplexor->sendCRC32MapRequest(mit->first,mHash); - found = true ; - break ; - } - } - break ; - - case FT_TM_CRC_MAP_STATE_HAVE: - { -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): got a CRC32 map. Matching chunks..." << std::endl ; -#endif - // The CRCmap is complete. Let's cross check it with existing chunks in ftFileCreator ! - // - uint32_t bad_chunks ; - uint32_t incomplete_chunks ; - - if(!mFileCreator->crossCheckChunkMap(_crcmap,bad_chunks,incomplete_chunks)) - { - std::cerr << "ftTransferModule::checkCRC(): could not check CRC chunks!" << std::endl ; - return false; - } - - // go back to download stage, if not all chunks are correct. - // - if(bad_chunks > 0) - { - mFlag = FT_TM_FLAG_DOWNLOADING ; - mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING; - std::cerr << "ftTransferModule::checkCRC(): Done. File has errors: " << bad_chunks << " bad chunks found. Restarting download for these chunks only." << std::endl ; - } - else if(incomplete_chunks > 0) - { - mFlag = FT_TM_FLAG_DOWNLOADING ; - mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING; -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): Done. all chunks ok. Continuing download for remaining chunks." << std::endl ; -#endif - } - else - { - // We do as if the file is not complete. This way, it finishes properly. - // - //mFlag = FT_TM_FLAG_COMPLETE ; // Transfer is complete. - mFlag = FT_TM_FLAG_DOWNLOADING ; - mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING; -#ifdef FT_DEBUG - std::cerr << "ftTransferModule::checkCRC(): Done. CRC check is ok, file is complete." << std::endl ; -#endif - } - - _crcmap_state = FT_TM_CRC_MAP_STATE_NOCHECK ; - _crcreq_source = "" ; - } - } - - return true ; -} - -void ftTransferModule::addCRC32Map(const CRC32Map& crc_map) -{ - RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ - - // Note: for now, we only send complete CRC32 maps, so the crc_map is always complete. When we - // send partial CRC maps, we will have to check them for completness. - // - _crcmap_state = FT_TM_CRC_MAP_STATE_HAVE ; - _crcmap = crc_map ; -} - void ftTransferModule::adjustSpeed() { RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index ea820e1b4..f32e9f150 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -38,7 +38,7 @@ GxsSecurity::~GxsSecurity() { } -RSA *GxsSecurity::extractPublicKey(RsTlvSecurityKey& key) +RSA *GxsSecurity::extractPublicKey(const RsTlvSecurityKey& key) { const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data; long keylen = key.keyData.bin_len; @@ -489,7 +489,7 @@ void GxsSecurity::setRSAPrivateKey(RsTlvSecurityKey & key, RSA *rsa_priv) key.keyId = keyId; } -RSA *GxsSecurity::extractPrivateKey(RsTlvSecurityKey & key) +RSA *GxsSecurity::extractPrivateKey(const RsTlvSecurityKey & key) { const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data; long keylen = key.keyData.bin_len; diff --git a/libretroshare/src/gxs/gxssecurity.h b/libretroshare/src/gxs/gxssecurity.h index 2bccf542f..4ebd122f4 100644 --- a/libretroshare/src/gxs/gxssecurity.h +++ b/libretroshare/src/gxs/gxssecurity.h @@ -51,14 +51,14 @@ public: * @param key RsTlvSecurityKey to extract public RSA key from * @return pointer to the public RSA key if successful, null otherwise */ - static RSA *extractPublicKey(RsTlvSecurityKey &key); + static RSA *extractPublicKey(const RsTlvSecurityKey &key); /*! * extracts the public key from an RsTlvSecurityKey * @param key RsTlvSecurityKey to extract private RSA key from * @return pointer to the private RSA key if successful, null otherwise */ - static RSA *extractPrivateKey(RsTlvSecurityKey &key); + static RSA *extractPrivateKey(const RsTlvSecurityKey &key); /*! * stores the rsa public key in a RsTlvSecurityKey diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 659b4b7dd..8a1626323 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -34,6 +34,8 @@ #define MSG_TABLE_NAME std::string("MESSAGES") #define GRP_TABLE_NAME std::string("GROUPS") +#define GRP_LAST_POST_UPDATE_TRIGGER std::string("LAST_POST_UPDATE") + // generic #define KEY_NXS_FILE std::string("nxsFile") @@ -42,12 +44,14 @@ #define KEY_NXS_IDENTITY std::string("identity") #define KEY_GRP_ID std::string("grpId") #define KEY_ORIG_GRP_ID std::string("origGrpId") +#define KEY_PARENT_GRP_ID std::string("parentGrpId") #define KEY_SIGN_SET std::string("signSet") #define KEY_TIME_STAMP std::string("timeStamp") #define KEY_NXS_FLAGS std::string("flags") #define KEY_NXS_META std::string("meta") #define KEY_NXS_SERV_STRING std::string("serv_str") #define KEY_NXS_HASH std::string("hash") +#define KEY_RECV_TS std::string("recv_time_stamp") // grp table columns @@ -111,6 +115,8 @@ #define COL_GRP_INTERN_CIRCLE 18 #define COL_GRP_ORIGINATOR 19 #define COL_GRP_AUTHEN_FLAGS 20 +#define COL_PARENT_GRP_ID 21 +#define COL_GRP_RECV_TS 22 // msg col numbers @@ -122,6 +128,7 @@ #define COL_THREAD_ID 11 #define COL_MSG_NAME 12 #define COL_MSG_SERV_STRING 13 +#define COL_MSG_RECV_TS 14 // generic meta shared col numbers #define COL_GRP_ID 0 @@ -154,7 +161,7 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d msgMetaColumns.push_back(KEY_SIGN_SET); msgMetaColumns.push_back(KEY_NXS_IDENTITY); msgMetaColumns.push_back(KEY_NXS_HASH); msgMetaColumns.push_back(KEY_MSG_ID); msgMetaColumns.push_back(KEY_ORIG_MSG_ID); msgMetaColumns.push_back(KEY_MSG_STATUS); msgMetaColumns.push_back(KEY_CHILD_TS); msgMetaColumns.push_back(KEY_MSG_PARENT_ID); msgMetaColumns.push_back(KEY_MSG_THREAD_ID); - msgMetaColumns.push_back(KEY_MSG_NAME); msgMetaColumns.push_back(KEY_NXS_SERV_STRING); + msgMetaColumns.push_back(KEY_MSG_NAME); msgMetaColumns.push_back(KEY_NXS_SERV_STRING); msgMetaColumns.push_back(KEY_RECV_TS); // for retrieving actual data msgColumns.push_back(KEY_GRP_ID); msgColumns.push_back(KEY_NXS_FILE); msgColumns.push_back(KEY_NXS_FILE_OFFSET); @@ -168,7 +175,8 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d grpMetaColumns.push_back(KEY_GRP_LAST_POST); grpMetaColumns.push_back(KEY_ORIG_GRP_ID); grpMetaColumns.push_back(KEY_NXS_SERV_STRING); grpMetaColumns.push_back(KEY_GRP_SIGN_FLAGS); grpMetaColumns.push_back(KEY_GRP_CIRCLE_ID); grpMetaColumns.push_back(KEY_GRP_CIRCLE_TYPE); grpMetaColumns.push_back(KEY_GRP_INTERNAL_CIRCLE); grpMetaColumns.push_back(KEY_GRP_ORIGINATOR); - grpMetaColumns.push_back(KEY_GRP_AUTHEN_FLAGS); + grpMetaColumns.push_back(KEY_GRP_AUTHEN_FLAGS); grpMetaColumns.push_back(KEY_PARENT_GRP_ID); grpMetaColumns.push_back(KEY_RECV_TS); + // for retrieving actual grp data grpColumns.push_back(KEY_GRP_ID); grpColumns.push_back(KEY_NXS_FILE); grpColumns.push_back(KEY_NXS_FILE_OFFSET); @@ -177,6 +185,8 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d // for retrieving msg offsets mMsgOffSetColumns.push_back(KEY_MSG_ID); mMsgOffSetColumns.push_back(KEY_NXS_FILE_OFFSET); mMsgOffSetColumns.push_back(KEY_NXS_FILE_LEN); + + grpIdColumn.push_back(KEY_GRP_ID); } RsDataService::~RsDataService(){ @@ -210,6 +220,7 @@ void RsDataService::initialise(){ KEY_MSG_NAME + " TEXT," + KEY_NXS_SERV_STRING + " TEXT," + KEY_NXS_HASH + " TEXT," + + KEY_RECV_TS + " INT," + KEY_NXS_FILE_LEN + " INT);"); // create table for grp data @@ -238,8 +249,16 @@ void RsDataService::initialise(){ KEY_GRP_INTERNAL_CIRCLE + " TEXT," + KEY_GRP_ORIGINATOR + " TEXT," + KEY_NXS_HASH + " TEXT," + + KEY_RECV_TS + " INT," + + KEY_PARENT_GRP_ID + " TEXT," + KEY_SIGN_SET + " BLOB);"); + mDb->execSQL("CREATE TRIGGER " + GRP_LAST_POST_UPDATE_TRIGGER + + " INSERT ON " + MSG_TABLE_NAME + + std::string(" BEGIN ") + + " UPDATE " + GRP_TABLE_NAME + " SET " + KEY_GRP_LAST_POST + "= new." + + KEY_RECV_TS + " WHERE " + KEY_GRP_ID + "=new." + KEY_GRP_ID + ";" + + std::string("END;")); } RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c) @@ -291,8 +310,11 @@ RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c) c.getString(COL_GRP_INTERN_CIRCLE, grpMeta->mInternalCircle); c.getString(COL_GRP_ORIGINATOR, grpMeta->mOriginator); grpMeta->mAuthenFlags = c.getInt32(COL_GRP_AUTHEN_FLAGS); + grpMeta->mRecvTS = c.getInt32(COL_GRP_RECV_TS); + c.getString(COL_PARENT_GRP_ID, grpMeta->mParentGrpId); + if(ok) return grpMeta; else @@ -377,6 +399,7 @@ RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c) c.getString(COL_MSG_NAME, msgMeta->mMsgName); c.getString(COL_MSG_SERV_STRING, msgMeta->mServiceString); c.getString(COL_HASH, msgMeta->mHash); + msgMeta->recvTS = c.getInt32(COL_MSG_RECV_TS); offset = 0; data = (char*)c.getData(COL_SIGN_SET, data_len); @@ -489,6 +512,7 @@ int RsDataService::storeMessage(std::map &msg) cv.put(KEY_GRP_ID, msgMetaPtr->mGroupId); cv.put(KEY_NXS_SERV_STRING, msgMetaPtr->mServiceString); cv.put(KEY_NXS_HASH, msgMetaPtr->mHash); + cv.put(KEY_RECV_TS, (int32_t)msgMetaPtr->recvTS); char signSetData[msgMetaPtr->signSet.TlvSize()]; @@ -595,7 +619,9 @@ int RsDataService::storeGroup(std::map &grp) cv.put(KEY_GRP_INTERNAL_CIRCLE, grpMetaPtr->mInternalCircle); cv.put(KEY_GRP_ORIGINATOR, grpMetaPtr->mOriginator); cv.put(KEY_GRP_AUTHEN_FLAGS, (int32_t)grpMetaPtr->mAuthenFlags); + cv.put(KEY_PARENT_GRP_ID, grpMetaPtr->mParentGrpId); cv.put(KEY_NXS_HASH, grpMetaPtr->mHash); + cv.put(KEY_RECV_TS, (int32_t)grpMetaPtr->mRecvTS); if(! (grpMetaPtr->mAuthorId.empty()) ){ cv.put(KEY_NXS_IDENTITY, grpMetaPtr->mAuthorId); @@ -641,6 +667,98 @@ int RsDataService::storeGroup(std::map &grp) return ret; } +int RsDataService::updateGroup(std::map &grp) +{ + + RsStackMutex stack(mDbMutex); + + std::map::iterator sit = grp.begin(); + + // begin transaction + mDb->execSQL("BEGIN;"); + + for(; sit != grp.end(); sit++) + { + + RsNxsGrp* grpPtr = sit->first; + RsGxsGrpMetaData* grpMetaPtr = sit->second; + + // if data is larger than max item size do not add + if(!validSize(grpPtr)) continue; + + std::string grpFile = mServiceDir + "/" + grpPtr->grpId; + std::ofstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::trunc); + uint32_t offset = 0; // get file offset + + /*! + * STORE file offset, file length, file name, + * grpId, flags, publish time stamp, identity, + * id signature, admin signatue, key set, last posting ts + * and meta data + **/ + ContentValue cv; + cv.put(KEY_NXS_FILE_OFFSET, (int32_t)offset); + cv.put(KEY_NXS_FILE_LEN, (int32_t)grpPtr->grp.TlvSize()); + cv.put(KEY_NXS_FILE, grpFile); + cv.put(KEY_GRP_ID, grpPtr->grpId); + cv.put(KEY_GRP_NAME, grpMetaPtr->mGroupName); + cv.put(KEY_ORIG_GRP_ID, grpMetaPtr->mOrigGrpId); + cv.put(KEY_NXS_SERV_STRING, grpMetaPtr->mServiceString); + cv.put(KEY_NXS_FLAGS, (int32_t)grpMetaPtr->mGroupFlags); + cv.put(KEY_TIME_STAMP, (int32_t)grpMetaPtr->mPublishTs); + cv.put(KEY_GRP_SIGN_FLAGS, (int32_t)grpMetaPtr->mSignFlags); + cv.put(KEY_GRP_CIRCLE_ID, grpMetaPtr->mCircleId); + cv.put(KEY_GRP_CIRCLE_TYPE, (int32_t)grpMetaPtr->mCircleType); + cv.put(KEY_GRP_INTERNAL_CIRCLE, grpMetaPtr->mInternalCircle); + cv.put(KEY_GRP_ORIGINATOR, grpMetaPtr->mOriginator); + cv.put(KEY_GRP_AUTHEN_FLAGS, (int32_t)grpMetaPtr->mAuthenFlags); + cv.put(KEY_NXS_HASH, grpMetaPtr->mHash); + + if(! (grpMetaPtr->mAuthorId.empty()) ){ + cv.put(KEY_NXS_IDENTITY, grpMetaPtr->mAuthorId); + } + + offset = 0; + char keySetData[grpMetaPtr->keys.TlvSize()]; + grpMetaPtr->keys.SetTlv(keySetData, grpMetaPtr->keys.TlvSize(), &offset); + cv.put(KEY_KEY_SET, grpMetaPtr->keys.TlvSize(), keySetData); + + offset = 0; + char metaData[grpPtr->meta.TlvSize()]; + grpPtr->meta.SetTlv(metaData, grpPtr->meta.TlvSize(), &offset); + cv.put(KEY_NXS_META, grpPtr->meta.TlvSize(), metaData); + + // local meta data + cv.put(KEY_GRP_SUBCR_FLAG, (int32_t)grpMetaPtr->mSubscribeFlags); + cv.put(KEY_GRP_POP, (int32_t)grpMetaPtr->mPop); + cv.put(KEY_MSG_COUNT, (int32_t)grpMetaPtr->mMsgCount); + cv.put(KEY_GRP_STATUS, (int32_t)grpMetaPtr->mGroupStatus); + cv.put(KEY_GRP_LAST_POST, (int32_t)grpMetaPtr->mLastPost); + + offset = 0; + char grpData[grpPtr->grp.TlvSize()]; + grpPtr->grp.SetTlv(grpData, grpPtr->grp.TlvSize(), &offset); + ostrm.write(grpData, grpPtr->grp.TlvSize()); + ostrm.close(); + + mDb->sqlUpdate(GRP_TABLE_NAME, "grpId='" + grpPtr->grpId + "'", cv); + } + // finish transaction + bool ret = mDb->execSQL("COMMIT;"); + + for(sit = grp.begin(); sit != grp.end(); sit++) + { + //TODO: API encourages aliasing, remove this abomination + if(sit->second != sit->first->metaData) + delete sit->second; + delete sit->first; + + } + + return ret; +} + + bool RsDataService::validSize(RsNxsGrp* grp) const { if((grp->grp.TlvSize() + grp->meta.TlvSize()) <= GXS_MAX_ITEM_SIZE) return true; @@ -964,34 +1082,33 @@ int RsDataService::retrieveGxsGrpMetaData(std::map::iterator mit = grp.begin(); - for(; mit != grp.end(); mit++) - { - const RsGxsGroupId& grpId = mit->first; - RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpMetaColumns, "grpId='" + grpId + "'", ""); + for(; mit != grp.end(); mit++) + { + const RsGxsGroupId& grpId = mit->first; + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpMetaColumns, "grpId='" + grpId + "'", ""); - if(c) - { - bool valid = c->moveToFirst(); + if(c) + { + bool valid = c->moveToFirst(); - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c); + while(valid) + { + RsGxsGrpMetaData* g = locked_getGrpMeta(*c); - if(g) - { - grp[g->mGroupId] = g; - } - valid = c->moveToNext(); - } - delete c; - } + if(g) + { + grp[g->mGroupId] = g; + } + valid = c->moveToNext(); + } + delete c; + } - } + } - } + } return 1; @@ -1010,21 +1127,22 @@ int RsDataService::resetDataStore() std::map::iterator mit = grps.begin(); - - // remove all grp msgs files from service dir - for(; mit != grps.end(); mit++){ - std::string file = mServiceDir + "/" + mit->first; - std::string msgFile = file + "-msgs"; - remove(file.c_str()); // remove group file - remove(msgFile.c_str()); // and remove messages file - delete mit->second; - } { RsStackMutex stack(mDbMutex); - mDb->closeDb(); - } - remove(mDbName.c_str()); // remove db file + // remove all grp msgs files from service dir + for(; mit != grps.end(); mit++){ + std::string file = mServiceDir + "/" + mit->first; + std::string msgFile = file + "-msgs"; + remove(file.c_str()); // remove group file + remove(msgFile.c_str()); // and remove messages file + delete mit->second; + } + + mDb->execSQL("DROP TABLE " + MSG_TABLE_NAME); + mDb->execSQL("DROP TABLE " + GRP_TABLE_NAME); + mDb->execSQL("DROP TRIGGER " + GRP_LAST_POST_UPDATE_TRIGGER); + } // recreate database initialise(); @@ -1177,6 +1295,31 @@ int RsDataService::removeGroups(const std::vector &grpIds) return 1; } +int RsDataService::retrieveGroupIds(std::vector &grpIds) +{ + RsStackMutex stack(mDbMutex); + + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpIdColumn, "", ""); + + if(c) + { + bool valid = c->moveToFirst(); + + while(valid) + { + std::string grpId; + c->getString(0, grpId); + grpIds.push_back(grpId); + valid = c->moveToNext(); + } + delete c; + }else + { + return 0; + } + + return 1; +} bool RsDataService::locked_updateMessageEntries(const MsgUpdates& updates) { diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index e649c59bb..a0fab25bd 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -106,6 +106,13 @@ public: */ int removeGroups(const std::vector& grpIds); + /*! + * Retrieves all group ids in store + * @param grpIds all grpids in store is inserted into this vector + * @return error code + */ + int retrieveGroupIds(std::vector &grpIds); + /*! * @return the cache size set for this RsGeneralDataService in bytes */ @@ -130,6 +137,13 @@ public: */ int storeGroup(std::map& grp); + /*! + * Updates group entries in Db + * @param grp map of group and decoded meta data + * @return error code + */ + int updateGroup(std::map& grsp); + /*! * @param metaData The meta data item to update * @return error code @@ -238,6 +252,7 @@ private: std::list grpColumns; std::list grpMetaColumns; + std::list grpIdColumn; std::string mServiceDir, mDbName; uint16_t mServType; diff --git a/libretroshare/src/gxs/rsgds.h b/libretroshare/src/gxs/rsgds.h index c7a5cf387..c2ba4d8ac 100644 --- a/libretroshare/src/gxs/rsgds.h +++ b/libretroshare/src/gxs/rsgds.h @@ -169,6 +169,13 @@ public: */ virtual int removeGroups(const std::vector& grpIds) = 0; + /*! + * Retrieves all group ids in store + * @param grpIds all grpids in store is inserted into this vector + * @return error code + */ + virtual int retrieveGroupIds(std::vector& grpIds) = 0; + /*! * @return the cache size set for this RsGeneralDataService in bytes */ @@ -194,6 +201,13 @@ public: virtual int storeGroup(std::map& grsp) = 0; + /*! + * Updates group entries in Db + * @param grp map of group and decoded meta data + * @return error code + */ + virtual int updateGroup(std::map& grsp) = 0; + /*! * @param metaData */ diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 5133d79ca..48778064f 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -26,11 +26,6 @@ #include -#include -#include -#include -#include - #include "pqi/pqihash.h" #include "rsgenexchange.h" #include "gxssecurity.h" @@ -118,6 +113,8 @@ void RsGenExchange::tick() publishMsgs(); + processGroupUpdatePublish(); + processRecvdData(); if(!mNotifications.empty()) @@ -278,6 +275,56 @@ void RsGenExchange::generateGroupKeys(RsTlvSecurityKeySet& privatekeySet, } } +void RsGenExchange::generatePublicFromPrivateKeys(const RsTlvSecurityKeySet &privatekeySet, + RsTlvSecurityKeySet &publickeySet) +{ + + // actually just copy settings of one key except mark its key flags public + + typedef std::map keyMap; + const keyMap& allKeys = privatekeySet.keys; + keyMap::const_iterator cit = allKeys.begin(); + bool adminFound = false, publishFound = false; + for(; cit != allKeys.end(); cit++) + { + const RsTlvSecurityKey& privKey = cit->second; + if(privKey.keyFlags & RSTLV_KEY_TYPE_FULL) + { + RsTlvSecurityKey pubKey; + + pubKey = privKey; + + RSA *rsaPrivKey = NULL, *rsaPubKey = NULL; + + rsaPrivKey = GxsSecurity::extractPrivateKey(privKey); + + if(rsaPrivKey) + rsaPubKey = RSAPublicKey_dup(rsaPrivKey); + + if(rsaPrivKey && rsaPubKey) + { + GxsSecurity::setRSAPublicKey(pubKey, rsaPubKey); + + if(pubKey.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) + pubKey.keyFlags = RSTLV_KEY_DISTRIB_ADMIN | RSTLV_KEY_TYPE_PUBLIC_ONLY; + + if(pubKey.keyFlags & RSTLV_KEY_DISTRIB_PRIVATE) + pubKey.keyFlags = RSTLV_KEY_DISTRIB_PRIVATE | RSTLV_KEY_TYPE_PUBLIC_ONLY; + + pubKey.endTS = pubKey.startTS + 60 * 60 * 24 * 365 * 5; /* approx 5 years */ + + publickeySet.keys.insert(std::make_pair(pubKey.keyId, pubKey)); + } + + if(rsaPrivKey) + RSA_free(rsaPrivKey); + + if(rsaPubKey) + RSA_free(rsaPubKey); + } + } +} + uint8_t RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet) { std::cerr << "RsGenExchange::createGroup()"; @@ -768,6 +815,7 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, RsTlvSecu }else { std::list peers; + peers.push_back(msg->PeerId()); mGixs->requestKey(metaData.mAuthorId, peers); return VALIDATE_FAIL_TRY_LATER; } @@ -841,6 +889,7 @@ int RsGenExchange::validateGrp(RsNxsGrp* grp, RsTlvSecurityKeySet& grpKeySet) }else { std::list peers; + peers.push_back(grp->PeerId()); mGixs->requestKey(metaData.mAuthorId, peers); return VALIDATE_FAIL_TRY_LATER; } @@ -1369,6 +1418,20 @@ void RsGenExchange::publishGroup(uint32_t& token, RsGxsGrpItem *grpItem) } + +void RsGenExchange::updateGroup(uint32_t& token, RsGxsGroupUpdateMeta& updateMeta, RsGxsGrpItem* grpItem) +{ + RsStackMutex stack(mGenMtx); + token = mDataAccess->generatePublicToken(); + mGroupUpdatePublish.push_back(GroupUpdatePublish(grpItem, updateMeta, token)); + +#ifdef GEN_EXCH_DEBUG + std::cerr << "RsGenExchange::updateGroup() token: " << token; + std::cerr << std::endl; +#endif +} + + void RsGenExchange::publishMsg(uint32_t& token, RsGxsMsgItem *msgItem) { RsStackMutex stack(mGenMtx); @@ -1721,6 +1784,7 @@ void RsGenExchange::publishMsgs() msg->metaData->mMsgStatus = GXS_SERV::GXS_MSG_STATUS_UNPROCESSED | GXS_SERV::GXS_MSG_STATUS_UNREAD; msgId = msg->msgId; grpId = msg->grpId; + msg->metaData->recvTS = time(NULL); computeHash(msg->msg, msg->metaData->mHash); mDataAccess->addMsgData(msg); msgChangeMap[grpId].push_back(msgId); @@ -1785,6 +1849,115 @@ RsGenExchange::ServiceCreate_Return RsGenExchange::service_CreateGroup(RsGxsGrpI #define PENDING_SIGN_TIMEOUT 10 // 5 seconds + +void RsGenExchange::processGroupUpdatePublish() +{ + + RsStackMutex stack(mGenMtx); + + // get keys for group update publish + + // first build meta request map for groups to be updated + std::map grpMeta; + std::vector::iterator vit = mGroupUpdatePublish.begin(); + + for(; vit != mGroupUpdatePublish.end(); vit++) + { + GroupUpdatePublish& gup = *vit; + const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId; + grpMeta.insert(std::make_pair(groupId, (RsGxsGrpMetaData*)(NULL))); + } + + if(grpMeta.empty()) + return; + + mDataStore->retrieveGxsGrpMetaData(grpMeta); + + // now + vit = mGroupUpdatePublish.begin(); + for(; vit != mGroupUpdatePublish.end(); vit++) + { + GroupUpdatePublish& gup = *vit; + const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId; + std::map::iterator mit = grpMeta.find(groupId); + + RsGxsGrpMetaData* meta = NULL; + if(mit == grpMeta.end()) + { + std::cerr << "Error! could not find meta of old group to update!" << std::endl; + mDataAccess->updatePublicRequestStatus(gup.mToken, RsTokenService::GXS_REQUEST_V2_STATUS_FAILED); + delete gup.grpItem; + continue; + }else + { + meta = mit->second; + } + + + gup.grpItem->meta = *meta; + assignMetaUpdates(gup.grpItem->meta, gup.mUpdateMeta); + GxsGrpPendingSign ggps(gup.grpItem, ggps.mToken); + + bool publishAndAdminPrivatePresent = checkKeys(meta->keys); + + if(publishAndAdminPrivatePresent) + { + ggps.mPrivateKeys = meta->keys; + generatePublicFromPrivateKeys(ggps.mPrivateKeys, ggps.mPublicKeys); + ggps.mHaveKeys = true; + ggps.mStartTS = time(NULL); + ggps.mLastAttemptTS = 0; + ggps.mIsUpdate = true; + ggps.mToken = gup.mToken; + mGrpsToPublish.push_back(ggps); + }else + { + delete gup.grpItem; + mDataAccess->updatePublicRequestStatus(gup.mToken, RsTokenService::GXS_REQUEST_V2_STATUS_FAILED); + } + delete meta; + } + + mGroupUpdatePublish.clear(); +} + +void RsGenExchange::assignMetaUpdates(RsGroupMetaData& meta, const RsGxsGroupUpdateMeta metaUpdate) const +{ + const RsGxsGroupUpdateMeta::GxsMetaUpdate* updates = metaUpdate.getUpdates(); + RsGxsGroupUpdateMeta::GxsMetaUpdate::const_iterator mit = updates->begin(); + for(; mit != updates->end(); mit++) + { + + if(mit->first == RsGxsGroupUpdateMeta::NAME) + meta.mGroupName = mit->second; + } +} + +bool RsGenExchange::checkKeys(const RsTlvSecurityKeySet& keySet) +{ + + typedef std::map keyMap; + const keyMap& allKeys = keySet.keys; + keyMap::const_iterator cit = allKeys.begin(); + bool adminFound = false, publishFound = false; + for(; cit != allKeys.end(); cit++) + { + const RsTlvSecurityKey& key = cit->second; + if(key.keyFlags & RSTLV_KEY_TYPE_FULL) + { + if(key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) + adminFound = true; + + if(key.keyFlags & RSTLV_KEY_DISTRIB_PRIVATE) + publishFound = true; + + } + } + + // user must have both private and public parts of publish and admin keys + return adminFound && publishFound; +} + void RsGenExchange::publishGrps() { RsStackMutex stack(mGenMtx); @@ -1855,10 +2028,6 @@ void RsGenExchange::publishGrps() // get group id from private admin key id grpItem->meta.mGroupId = grp->grpId = privAdminKey.keyId; - // what!? this will remove the private keys! - privatekeySet.keys.insert(publicKeySet.keys.begin(), - publicKeySet.keys.end()); - ServiceCreate_Return ret = service_CreateGroup(grpItem, privatekeySet); @@ -1882,6 +2051,7 @@ void RsGenExchange::publishGrps() { grp->metaData = new RsGxsGrpMetaData(); grpItem->meta.mPublishTs = time(NULL); + //grpItem->meta.mParentGrpId = std::string("empty"); *(grp->metaData) = grpItem->meta; // TODO: change when publish key optimisation added (public groups don't have publish key @@ -1906,7 +2076,12 @@ void RsGenExchange::publishGrps() { grpId = grp->grpId; computeHash(grp->grp, grp->metaData->mHash); - mDataAccess->addGroupData(grp); + grp->metaData->mRecvTS = time(NULL); + + if(ggps.mIsUpdate) + mDataAccess->updateGroupData(grp); + else + mDataAccess->addGroupData(grp); } else { @@ -2038,6 +2213,8 @@ void RsGenExchange::processRecvdData() processRecvdGroups(); processRecvdMessages(); + + performUpdateValidation(); } @@ -2070,6 +2247,9 @@ void RsGenExchange::processRecvdMessages() } } + if(mReceivedMsgs.empty()) + return; + std::vector::iterator vit = mReceivedMsgs.begin(); GxsMsgReq msgIds; std::map msgs; @@ -2122,6 +2302,7 @@ void RsGenExchange::processRecvdMessages() if(validated_entry != mMsgPendingValidate.end()) mMsgPendingValidate.erase(validated_entry); computeHash(msg->msg, meta->mHash); + meta->recvTS = time(NULL); } } else @@ -2192,10 +2373,16 @@ void RsGenExchange::processRecvdGroups() { RsStackMutex stack(mGenMtx); + if(mReceivedGrps.empty()) + return; + NxsGrpPendValidVect::iterator vit = mReceivedGrps.begin(); + std::vector existingGrpIds; + std::list grpIds; + std::map grps; - std::list grpIds; + mDataStore->retrieveGroupIds(existingGrpIds); while( vit != mReceivedGrps.end()) { @@ -2220,13 +2407,24 @@ void RsGenExchange::processRecvdGroups() meta->mGroupStatus = GXS_SERV::GXS_GRP_STATUS_UNPROCESSED | GXS_SERV::GXS_GRP_STATUS_UNREAD; meta->mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED; - if(meta->mCircleType == GXS_CIRCLE_TYPE_YOUREYESONLY) - meta->mOriginator = grp->PeerId(); - computeHash(grp->grp, meta->mHash); - grps.insert(std::make_pair(grp, meta)); - grpIds.push_back(grp->grpId); + // now check if group already existss + if(std::find(existingGrpIds.begin(), existingGrpIds.end(), grp->grpId) == existingGrpIds.end()) + { + meta->mRecvTS = time(NULL); + if(meta->mCircleType == GXS_CIRCLE_TYPE_YOUREYESONLY) + meta->mOriginator = grp->PeerId(); + + grps.insert(std::make_pair(grp, meta)); + grpIds.push_back(grp->grpId); + } + else + { + GroupUpdate update; + update.newGrp = grp; + mGroupUpdates.push_back(update); + } erase = true; } else if(ret == VALIDATE_FAIL) @@ -2278,3 +2476,99 @@ void RsGenExchange::processRecvdGroups() mDataStore->storeGroup(grps); } } + +void RsGenExchange::performUpdateValidation() +{ + + RsStackMutex stack(mGenMtx); + +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl; +#endif + + if(mGroupUpdates.empty()) + return; + + std::map grpMetas; + + std::vector::iterator vit = mGroupUpdates.begin(); + for(; vit != mGroupUpdates.end(); vit++) + grpMetas.insert(std::make_pair(vit->newGrp->grpId, (RsGxsGrpMetaData*)NULL)); + + if(!grpMetas.empty()) + mDataStore->retrieveGxsGrpMetaData(grpMetas); + else + return; + + vit = mGroupUpdates.begin(); + for(; vit != mGroupUpdates.end(); vit++) + { + GroupUpdate& gu = *vit; + std::map::iterator mit = + grpMetas.find(gu.newGrp->grpId); + gu.oldGrpMeta = mit->second; + gu.validUpdate = updateValid(*(gu.oldGrpMeta), *(gu.newGrp)); + } + +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl; +#endif + + vit = mGroupUpdates.begin(); + std::map grps; + for(; vit != mGroupUpdates.end(); vit++) + { + GroupUpdate& gu = *vit; + + if(gu.validUpdate) + { + if(gu.newGrp->metaData->mCircleType == GXS_CIRCLE_TYPE_YOUREYESONLY) + gu.newGrp->metaData->mOriginator = gu.newGrp->PeerId(); + + grps.insert(std::make_pair(gu.newGrp, gu.newGrp->metaData)); + } + else + { + delete gu.newGrp; + } + + delete gu.oldGrpMeta; + } + + mDataStore->updateGroup(grps); + mGroupUpdates.clear(); +} + +bool RsGenExchange::updateValid(RsGxsGrpMetaData& oldGrpMeta, RsNxsGrp& newGrp) const +{ + std::map& signSet = newGrp.metaData->signSet.keySignSet; + std::map::iterator mit = signSet.find(GXS_SERV::FLAG_AUTHEN_ADMIN); + + if(mit == signSet.end()) + { +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::updateValid() new admin sign not found! " << std::endl; + std::cerr << "RsGenExchange::updateValid() grpId: " << oldGrp.grpId << std::endl; +#endif + + return false; + } + + RsTlvKeySignature adminSign = mit->second; + + std::map& keys = oldGrpMeta.keys.keys; + std::map::iterator keyMit = keys.find(oldGrpMeta.mGroupId); + + if(keyMit == keys.end()) + { +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::updateValid() admin key not found! " << std::endl; +#endif + return false; + } + + // also check this is the latest published group + bool latest = newGrp.metaData->mPublishTs > oldGrpMeta.mPublishTs; + + return GxsSecurity::validateNxsGrp(newGrp, adminSign, keyMit->second) && latest; +} diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index a9d3d7b4a..b495d7922 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -71,13 +71,14 @@ class GxsGrpPendingSign public: GxsGrpPendingSign(RsGxsGrpItem* item, uint32_t token): mLastAttemptTS(0), mStartTS(time(NULL)), mToken(token), - mItem(item), mHaveKeys(false) + mItem(item), mHaveKeys(false), mIsUpdate(false) {} time_t mLastAttemptTS, mStartTS; uint32_t mToken; RsGxsGrpItem* mItem; bool mHaveKeys; // mKeys->first == true if key present + bool mIsUpdate; RsTlvSecurityKeySet mPrivateKeys; RsTlvSecurityKeySet mPublicKeys; }; @@ -516,7 +517,6 @@ protected: /*! * Enables publication of a group item \n - * If the item exists already this is simply versioned \n * This will induce a related change message \n * Ownership of item passes to this rsgenexchange \n * @param token @@ -524,6 +524,16 @@ protected: */ void publishGroup(uint32_t& token, RsGxsGrpItem* grpItem); + + /*! + * Updates an existing group item \n + * This will induce a related change message \n + * Ownership of item passes to this rsgenexchange \n + * @param token + * @param grpItem + */ + void updateGroup(uint32_t& token, RsGxsGroupUpdateMeta& updateMeta, RsGxsGrpItem* grpItem); + public: /*! * Enables publication of a message item \n @@ -629,6 +639,8 @@ private: void publishGrps(); + void processGroupUpdatePublish(); + void publishMsgs(); /*! @@ -700,11 +712,20 @@ private: /*! * Generate a set of keys that can define a GXS group * @param privatekeySet contains private generated keys - * @param privatekeySet contains public generated keys (counterpart of private) + * @param publickeySet contains public generated keys (counterpart of private) * @param genPublicKeys should publish key pair also be generated */ void generateGroupKeys(RsTlvSecurityKeySet& privatekeySet, RsTlvSecurityKeySet& publickeySet, bool genPublishKeys); + /*! + * Generate public set of keys from their private counterparts + * No keys will be generated if one fails + * @param privatekeySet contains private generated keys + * @param publickeySet contains public generated keys (counterpart of private) + * @return false if key gen failed for a key set + */ + void generatePublicFromPrivateKeys(const RsTlvSecurityKeySet& privatekeySet, RsTlvSecurityKeySet& publickeySet); + /*! * Attempts to validate msg signatures * @param msg message to be validated @@ -736,6 +757,32 @@ private: static void computeHash(const RsTlvBinaryData& data, std::string& hash); + /*! + * Checks validation of recently received groups to be + * updated + */ + void performUpdateValidation(); + + /*! + * Checks if the update is valid (i.e. the new admin signature is by the old admin key) + * @param oldGrp the old group to be updated (must have meta data member initialised) + * @param newGrp the new group that updates the old group (must have meta data member initialised) + * @return + */ + bool updateValid(RsGxsGrpMetaData& oldGrp, RsNxsGrp& newGrp) const; + + /*! + * convenience function for checking private publish and admin keys are present + * @param keySet The keys set to split into a private and public set + * @return false, if private admin and publish keys cannot be found, true otherwise + */ + bool checkKeys(const RsTlvSecurityKeySet& keySet); + + /*! + * Convenience function for assigning the meta update items to the actual group meta + */ + void assignMetaUpdates(RsGroupMetaData& meta, const RsGxsGroupUpdateMeta metaUpdate) const; + private: RsMutex mGenMtx; @@ -798,6 +845,13 @@ private: const uint8_t CREATE_FAIL, CREATE_SUCCESS, CREATE_FAIL_TRY_LATER, SIGN_MAX_ATTEMPTS; const uint8_t SIGN_FAIL, SIGN_SUCCESS, SIGN_FAIL_TRY_LATER; const uint8_t VALIDATE_FAIL, VALIDATE_SUCCESS, VALIDATE_FAIL_TRY_LATER, VALIDATE_MAX_ATTEMPTS; + +private: + + std::vector mGroupUpdates, mPeersGroupUpdate; + + std::vector mGroupUpdatePublish; + }; #endif // RSGENEXCHANGE_H diff --git a/libretroshare/src/gxs/rsgixs.h b/libretroshare/src/gxs/rsgixs.h index 304609711..cd3bddf9a 100644 --- a/libretroshare/src/gxs/rsgixs.h +++ b/libretroshare/src/gxs/rsgixs.h @@ -176,7 +176,7 @@ class RsGixsReputation public: // get Reputation. virtual bool haveReputation(const RsGxsId &id) = 0; - virtual bool loadReputation(const RsGxsId &id) = 0; + virtual bool loadReputation(const RsGxsId &id, const std::list& peers) = 0; virtual bool getReputation(const RsGxsId &id, GixsReputation &rep) = 0; }; diff --git a/libretroshare/src/gxs/rsgxsdata.cc b/libretroshare/src/gxs/rsgxsdata.cc index fbccd8af2..45f4db18f 100644 --- a/libretroshare/src/gxs/rsgxsdata.cc +++ b/libretroshare/src/gxs/rsgxsdata.cc @@ -49,6 +49,7 @@ uint32_t RsGxsGrpMetaData::serial_size() s += 4; // for mCircleType s += GetTlvStringSize(mCircleId); s += 4; // mAuthenFlag + s += GetTlvStringSize(mParentGrpId); return s; } @@ -74,6 +75,8 @@ void RsGxsGrpMetaData::clear(){ mOriginator.clear(); mCircleType = 0; mAuthenFlags = 0; + mParentGrpId.clear(); + mRecvTS = 0; } @@ -103,6 +106,7 @@ bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize) ok &= SetTlvString(data, tlvsize, &offset, 0, mGroupId); ok &= SetTlvString(data, tlvsize, &offset, 0, mOrigGrpId); + ok &= SetTlvString(data, tlvsize, &offset, 0, mParentGrpId); ok &= SetTlvString(data, tlvsize, &offset, 0, mGroupName); ok &= setRawUInt32(data, tlvsize, &offset, mGroupFlags); ok &= setRawUInt32(data, tlvsize, &offset, mPublishTs); @@ -133,6 +137,7 @@ bool RsGxsGrpMetaData::deserialise(void *data, uint32_t &pktsize) ok &= GetTlvString(data, pktsize, &offset, 0, mGroupId); ok &= GetTlvString(data, pktsize, &offset, 0, mOrigGrpId); + ok &= GetTlvString(data, pktsize, &offset, 0, mParentGrpId); ok &= GetTlvString(data, pktsize, &offset, 0, mGroupName); ok &= getRawUInt32(data, pktsize, &offset, &mGroupFlags); ok &= getRawUInt32(data, pktsize, &offset, &mPublishTs); @@ -196,6 +201,7 @@ void RsGxsMsgMetaData::clear() mMsgFlags = 0; mMsgStatus = 0; mChildTs = 0; + recvTS = 0; } bool RsGxsMsgMetaData::serialise(void *data, uint32_t *size) @@ -284,6 +290,8 @@ void RsGxsGrpMetaData::operator =(const RsGroupMetaData& rMeta) this->mInternalCircle = rMeta.mInternalCircle; this->mOriginator = rMeta.mOriginator; this->mAuthenFlags = rMeta.mAuthenFlags; + //std::cout << "rMeta.mParentGrpId= " <mParentGrpId = rMeta.mParentGrpId; } void RsGxsMsgMetaData::operator =(const RsMsgMetaData& rMeta) diff --git a/libretroshare/src/gxs/rsgxsdata.h b/libretroshare/src/gxs/rsgxsdata.h index 9a94cc8a0..6fe905a9a 100644 --- a/libretroshare/src/gxs/rsgxsdata.h +++ b/libretroshare/src/gxs/rsgxsdata.h @@ -63,6 +63,7 @@ public: std::string mServiceString; uint32_t mAuthenFlags; + RsGxsGroupId mParentGrpId; // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG. @@ -70,17 +71,16 @@ public: uint32_t mPop; // HOW DO WE DO THIS NOW. uint32_t mMsgCount; // ??? - time_t mLastPost; // ??? + uint32_t mLastPost; // ??? uint32_t mGroupStatus; + uint32_t mRecvTS; std::string mOriginator; std::string mInternalCircle; std::string mHash; }; - - class RsGxsMsgMetaData { public: @@ -114,6 +114,7 @@ public: uint32_t mMsgStatus; time_t mChildTs; + uint32_t recvTS; std::string mHash; bool validated; diff --git a/libretroshare/src/gxs/rsgxsdataaccess.cc b/libretroshare/src/gxs/rsgxsdataaccess.cc index 2ac4e11a6..affea06e1 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.cc +++ b/libretroshare/src/gxs/rsgxsdataaccess.cc @@ -1489,7 +1489,14 @@ bool RsGxsDataAccess::addGroupData(RsNxsGrp* grp) { return mDataStore->storeGroup(grpM); } +bool RsGxsDataAccess::updateGroupData(RsNxsGrp* grp) { + RsStackMutex stack(mDataMutex); + + std::map grpM; + grpM.insert(std::make_pair(grp, grp->metaData)); + return mDataStore->updateGroup(grpM); +} bool RsGxsDataAccess::addMsgData(RsNxsMsg* msg) { diff --git a/libretroshare/src/gxs/rsgxsdataaccess.h b/libretroshare/src/gxs/rsgxsdataaccess.h index de5258569..1cf439209 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.h +++ b/libretroshare/src/gxs/rsgxsdataaccess.h @@ -126,14 +126,21 @@ public: public: /*! - * This adds a groups to the gxs data base, this is a blocking call - * Responsibility for grp still lies with callee \n + * This adds a groups to the gxs data base, this is a blocking call \n * If function returns successfully DataAccess can be queried for grp * @param grp the group to add, responsibility grp passed lies with callee * @return false if group cound not be added */ bool addGroupData(RsNxsGrp* grp); + /*! + * This updates a groups in the gxs data base, this is a blocking call \n + * If function returns successfully DataAccess can be queried for grp + * @param grp the group to add, responsibility grp passed lies with callee + * @return false if group cound not be added + */ + bool updateGroupData(RsNxsGrp* grp); + /*! * This adds a group to the gxs data base, this is a blocking call \n * Responsibility for msg still lies with callee \n @@ -429,11 +436,13 @@ private: private: RsGeneralDataService* mDataStore; + + RsMutex mDataMutex; /* protecting below */ + uint32_t mNextToken; std::map mPublicToken; std::map mRequests; - RsMutex mDataMutex; }; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 08e6739b4..a15336600 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -36,16 +36,16 @@ #define GIXS_CUT_OFF 0 #define SYNC_PERIOD 12 // in microseconds every 10 seconds (1 second for testing) -#define TRANSAC_TIMEOUT 5 // 5 seconds +#define TRANSAC_TIMEOUT 10 // 10 seconds const uint32_t RsGxsNetService::FRAGMENT_SIZE = 150000; RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, - RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, RsGixsReputation* reputations, RsGcxs* circles) + RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, RsGixsReputation* reputations, RsGcxs* circles, bool grpAutoSync) : p3Config(servType), p3ThreadedService(servType), mTransactionTimeOut(TRANSAC_TIMEOUT), mServType(servType), mDataStore(gds), mTransactionN(0), mObserver(nxsObs), mNxsMutex("RsGxsNetService"), mNetMgr(netMgr), mSYNC_PERIOD(SYNC_PERIOD), - mSyncTs(0), mReputations(reputations), mCircles(circles) + mSyncTs(0), mReputations(reputations), mCircles(circles), mGrpAutoSync(grpAutoSync), mGrpServerUpdateItem(NULL) { addSerialType(new RsNxsSerialiser(mServType)); @@ -85,13 +85,27 @@ void RsGxsNetService::syncWithPeers() std::set::iterator sit = peers.begin(); - // for now just grps - for(; sit != peers.end(); sit++) + if(mGrpAutoSync) { - RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); - grp->clear(); - grp->PeerId(*sit); - sendItem(grp); + // for now just grps + for(; sit != peers.end(); sit++) + { + + const std::string peerId = *sit; + + ClientGrpMap::const_iterator cit = mClientGrpUpdateMap.find(peerId); + uint32_t updateTS = 0; + if(cit != mClientGrpUpdateMap.end()) + { + const RsGxsGrpUpdateItem *gui = cit->second; + updateTS = gui->grpUpdateTS; + } + RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); + grp->clear(); + grp->PeerId(*sit); + grp->updateTS = updateTS; + sendItem(grp); + } } #ifdef GXS_ENABLE_SYNC_MSGS @@ -108,7 +122,9 @@ void RsGxsNetService::syncWithPeers() RsGxsGrpMetaData* meta = mit->second; if(meta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) + { grpIds.push_back(mit->first); + } delete meta; } @@ -122,13 +138,36 @@ void RsGxsNetService::syncWithPeers() std::vector::iterator vit = grpIds.begin(); + // now see if you have an updateTS so optimise whether you need + // to get a new list of peer data + RsGxsMsgUpdateItem* mui = NULL; + + ClientMsgMap::const_iterator cit = mClientMsgUpdateMap.find(*sit); + + if(cit != mClientMsgUpdateMap.end()) + { + mui = cit->second; + } + for(; vit != grpIds.end(); vit++) { - RsNxsSyncMsg* msg = new RsNxsSyncMsg(mServType); - msg->clear(); - msg->PeerId(*sit); - msg->grpId = *vit; - sendItem(msg); + uint32_t updateTS = 0; + if(mui) + { + std::map::const_iterator cit2 = mui->msgUpdateTS.find(*vit); + + if(cit2 != mui->msgUpdateTS.end()) + { + updateTS = cit2->second; + } + } + + RsNxsSyncMsg* msg = new RsNxsSyncMsg(mServType); + msg->clear(); + msg->PeerId(*sit); + msg->grpId = *vit; + msg->updateTS = updateTS; + sendItem(msg); } } #endif @@ -524,20 +563,105 @@ void RsGxsNetService::collateMsgFragments(MsgFragments fragments, std::map& load) +class StoreHere { - return false; +public: + + StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, + RsGxsNetService::ServerMsgMap& smm, + RsGxsServerGrpUpdateItem*& sgm) : mClientGrpMap(cgm), mClientMsgMap(cmm), + mServerMsgMap(smm), mServerGrpUpdateItem(sgm) + {} + + void operator() (RsItem* item) + { + RsGxsMsgUpdateItem* mui; + RsGxsGrpUpdateItem* gui; + RsGxsServerGrpUpdateItem* gsui; + RsGxsServerMsgUpdateItem* msui; + + if((mui = dynamic_cast(item)) != NULL) + mClientMsgMap.insert(std::make_pair(mui->peerId, mui)); + else if((gui = dynamic_cast(item)) != NULL) + mClientGrpMap.insert(std::make_pair(gui->peerId, gui)); + else if((msui = dynamic_cast(item)) != NULL) + mServerMsgMap.insert(std::make_pair(msui->grpId, msui)); + else if((gsui = dynamic_cast(item)) != NULL) + { + if(mServerGrpUpdateItem == NULL) + { + mServerGrpUpdateItem = gsui; + } + else + { +#ifdef NXS_NET_DEBUG + std::cerr << "Error! More than one server group update item exists!" << std::endl; +#endif + delete gsui; + } + } + else + { + std::cerr << "Type not expected!" << std::endl; + } + + } + +private: + + RsGxsNetService::ClientGrpMap& mClientGrpMap; + RsGxsNetService::ClientMsgMap& mClientMsgMap; + RsGxsNetService::ServerMsgMap& mServerMsgMap; + RsGxsServerGrpUpdateItem*& mServerGrpUpdateItem; + +}; + +bool RsGxsNetService::loadList(std::list &load) +{ + std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, + mServerMsgUpdateMap, mGrpServerUpdateItem)); + + return true; } +#include + +template +struct get_second : public std::unary_function +{ + RsItem* operator()(const typename UpdateMap::value_type& value) const + { + return value.second; + } +}; + bool RsGxsNetService::saveList(bool& cleanup, std::list& save) { - return false; + RsStackMutex stack(mNxsMutex); + + // hardcore templates + std::transform(mClientGrpUpdateMap.begin(), mClientGrpUpdateMap.end(), + std::back_inserter(save), get_second()); + + std::transform(mClientMsgUpdateMap.begin(), mClientMsgUpdateMap.end(), + std::back_inserter(save), get_second()); + + std::transform(mServerMsgUpdateMap.begin(), mServerMsgUpdateMap.end(), + std::back_inserter(save), get_second()); + + save.push_back(mGrpServerUpdateItem); + + cleanup = false; + return true; } RsSerialiser *RsGxsNetService::setupSerialiser() { - return NULL; + + RsSerialiser *rss = new RsSerialiser; + rss->addSerialType(new RsGxsUpdateSerialiser(mServType)); + + return rss; } void RsGxsNetService::recvNxsItemQueue(){ @@ -748,7 +872,8 @@ bool RsGxsNetService::locked_processTransac(RsNxsTransac* item) void RsGxsNetService::run(){ - double timeDelta = 0.2; + double timeDelta = 0.5; + int updateCounter = 0; while(isRunning()){ @@ -758,6 +883,14 @@ void RsGxsNetService::run(){ Sleep((int) (timeDelta * 1000)); #endif + if(updateCounter == 20) + { + updateServerSyncTS(); + updateCounter = 0; + } + else + updateCounter++; + // process active transactions processTransactions(); @@ -767,13 +900,71 @@ void RsGxsNetService::run(){ // vetting of id and circle info runVetting(); + processExplicitGroupRequests(); + } } +void RsGxsNetService::updateServerSyncTS() +{ + RsStackMutex stack(mNxsMutex); + + std::map gxsMap; + + // retrieve all grps and update TS + mDataStore->retrieveGxsGrpMetaData(gxsMap); + std::map::iterator mit = gxsMap.begin(); + + // as a grp list server also note this is the latest item you have + if(mGrpServerUpdateItem == NULL) + { + mGrpServerUpdateItem = new RsGxsServerGrpUpdateItem(mServType); + } + + bool change = false; + + for(; mit != gxsMap.end(); mit++) + { + const RsGxsGroupId& grpId = mit->first; + RsGxsGrpMetaData* grpMeta = mit->second; + ServerMsgMap::iterator mapIT = mServerMsgUpdateMap.find(grpId); + RsGxsServerMsgUpdateItem* msui = NULL; + + if(mapIT == mServerMsgUpdateMap.end()) + { + msui = new RsGxsServerMsgUpdateItem(mServType); + msui->grpId = grpMeta->mGroupId; + mServerMsgUpdateMap.insert(std::make_pair(msui->grpId, msui)); + }else + { + msui = mapIT->second; + } + + if(grpMeta->mLastPost > msui->msgUpdateTS ) + { + change = true; + msui->msgUpdateTS = grpMeta->mLastPost; + } + + // this might be very inefficient with time + if(grpMeta->mRecvTS > mGrpServerUpdateItem->grpUpdateTS) + { + mGrpServerUpdateItem->grpUpdateTS = grpMeta->mRecvTS; + change = true; + } + } + + // actual change in config settings, then save configuration + if(change) + IndicateConfigChanged(); + + freeAndClearContainerResource, + RsGxsGrpMetaData*>(gxsMap); + +} bool RsGxsNetService::locked_checkTransacTimedOut(NxsTransaction* tr) { - //return tr->mTimeOut < ((uint32_t) time(NULL)); - return false; + return tr->mTimeOut < ((uint32_t) time(NULL)); } void RsGxsNetService::processTransactions(){ @@ -1036,25 +1227,52 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) // notify listener of grps mObserver->notifyNewGroups(grps); + // now note this as the latest you've received from this peer + std::string peerFrom = tr->mTransaction->PeerId(); + uint32_t updateTS = tr->mTransaction->updateTS; + + ClientGrpMap::iterator it = mClientGrpUpdateMap.find(peerFrom); + + RsGxsGrpUpdateItem* item = NULL; + + if(it != mClientGrpUpdateMap.end()) + { + item = it->second; + }else + { + item = new RsGxsGrpUpdateItem(mServType); + mClientGrpUpdateMap.insert( + std::make_pair(peerFrom, item)); + } + + item->grpUpdateTS = updateTS; + item->peerId = peerFrom; + + IndicateConfigChanged(); + }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) { std::vector msgs; + std::string grpId; while(tr->mItems.size() > 0) { RsNxsMsg* msg = dynamic_cast(tr->mItems.front()); if(msg) { - tr->mItems.pop_front(); - msgs.push_back(msg); + if(grpId.empty()) + grpId = msg->grpId; + + tr->mItems.pop_front(); + msgs.push_back(msg); } else { #ifdef NXS_NET_DEBUG - std::cerr << "RsGxsNetService::processCompletedTransactions(): item did not caste to msg" - << std::endl; + std::cerr << "RsGxsNetService::processCompletedTransactions(): item did not caste to msg" + << std::endl; #endif } } @@ -1078,6 +1296,10 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) // notify listener of msgs mObserver->notifyNewMessages(msgs); + // now note that this is the latest you've received from this peer + // for the grp id + locked_doMsgUpdateWork(tr->mTransaction, grpId); + } }else if(tr->mFlag == NxsTransaction::FLAG_STATE_FAILED){ // don't do anything transaction will simply be cleaned @@ -1085,6 +1307,33 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) return; } +void RsGxsNetService::locked_doMsgUpdateWork(const RsNxsTransac *nxsTrans, const std::string &grpId) +{ + + // firts check if peer exists + const std::string& peerFrom = nxsTrans->PeerId(); + + ClientMsgMap::iterator it = mClientMsgUpdateMap.find(peerFrom); + + RsGxsMsgUpdateItem* mui = NULL; + + // now update the peer's entry for this grp id + if(it != mClientMsgUpdateMap.end()) + { + mui = it->second; + } + else + { + mui = new RsGxsMsgUpdateItem(mServType); + mClientMsgUpdateMap.insert(std::make_pair(peerFrom, mui)); + } + + mui->msgUpdateTS[grpId] = nxsTrans->updateTS; + mui->peerId = peerFrom; + + IndicateConfigChanged(); +} + void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) { uint16_t flag = tr->mTransaction->transactFlag; @@ -1125,6 +1374,7 @@ void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) std::cerr << "complete Sending Grp Data, transN: " << tr->mTransaction->transactionNumber << std::endl; #endif + }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) { #ifdef NXS_NET_DEBUG @@ -1166,7 +1416,7 @@ void RsGxsNetService::locked_pushMsgTransactionFromList( newTrans->mTimeOut = time(NULL) + mTransactionTimeOut; // create transaction copy with your id to indicate // its an outgoing transaction - newTrans->mTransaction = new RsNxsTransac(*transac); + newTrans->mTransaction = new RsNxsTransac(*transac); newTrans->mTransaction->PeerId(mOwnId); sendItem(transac); { @@ -1248,7 +1498,7 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) msgIdSet.insert((*vit)->mMsgId); delete(*vit); } - msgMetaV.clear(); + msgMetaV.clear(); // get unique id for this transaction uint32_t transN = locked_getTransactionId(); @@ -1261,6 +1511,9 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) MsgAuthorV toVet; + std::list peers; + peers.push_back(tr->mTransaction->PeerId()); + for(; llit != msgItemL.end(); llit++) { RsNxsSyncMsgItem*& syncItem = *llit; @@ -1294,7 +1547,7 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) else { // preload for speed - mReputations->loadReputation(syncItem->authorId); + mReputations->loadReputation(syncItem->authorId, peers); MsgAuthEntry entry; entry.mAuthorId = syncItem->authorId; entry.mGrpId = syncItem->grpId; @@ -1375,6 +1628,7 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) } std::map grpMetaMap; + std::map::const_iterator metaIter; mDataStore->retrieveGxsGrpMetaData(grpMetaMap); // now do compare and add loop @@ -1384,13 +1638,23 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) uint32_t transN = locked_getTransactionId(); GrpAuthorV toVet; + std::list peers; + peers.push_back(tr->mTransaction->PeerId()); for(; llit != grpItemL.end(); llit++) { RsNxsSyncGrpItem*& grpSyncItem = *llit; const std::string& grpId = grpSyncItem->grpId; + metaIter = grpMetaMap.find(grpId); + bool haveItem = false; + bool latestVersion = false; + if (metaIter != grpMetaMap.end()) + { + haveItem = true; + latestVersion = grpSyncItem->publishTs > metaIter->second->mPublishTs; + } - if(grpMetaMap.find(grpId) == grpMetaMap.end()){ + if(!haveItem || (haveItem && latestVersion) ){ // determine if you need to check reputation bool checkRep = !grpSyncItem->authorId.empty(); @@ -1412,7 +1676,7 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) else { // preload reputation for later - mReputations->loadReputation(grpSyncItem->authorId); + mReputations->loadReputation(grpSyncItem->authorId, peers); GrpAuthEntry entry; entry.mAuthorId = grpSyncItem->authorId; entry.mGrpId = grpSyncItem->grpId; @@ -1505,11 +1769,15 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) return; } + uint32_t updateTS = 0; + if(mGrpServerUpdateItem) + updateTS = mGrpServerUpdateItem->grpUpdateTS; RsNxsTransac* ntr = new RsNxsTransac(mServType); ntr->transactionNumber = transN; ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_GRPS; + ntr->updateTS = updateTS; ntr->nItems = grps.size(); ntr->PeerId(tr->mTransaction->PeerId()); @@ -1625,12 +1893,17 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) return; } + std::string grpId = ""; + for(;lit != tr->mItems.end(); lit++) { RsNxsSyncMsgItem* item = dynamic_cast(*lit); if (item) { msgIds[item->grpId].push_back(item->msgId); + + if(grpId.empty()) + grpId = item->grpId; } else { @@ -1687,10 +1960,18 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) return; } + uint32_t updateTS = 0; + + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(grpId); + + if(cit != mServerMsgUpdateMap.end()) + updateTS = cit->second->msgUpdateTS; + RsNxsTransac* ntr = new RsNxsTransac(mServType); ntr->transactionNumber = transN; ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_MSGS; + ntr->updateTS = updateTS; ntr->nItems = msgSize; ntr->PeerId(peerId); @@ -1766,13 +2047,32 @@ void RsGxsNetService::locked_pushGrpRespFromList(std::list& respList locked_addTransaction(tr); } +bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncGrp *item) +{ + // don't sync if you have no new updates for this peer + if(mGrpServerUpdateItem) + { + if(item->updateTS >= mGrpServerUpdateItem->grpUpdateTS && item->updateTS != 0) + { + return false; + } + } + + return true; +} + void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) { RsStackMutex stack(mNxsMutex); + if(!locked_CanReceiveUpdate(item)) + return; + std::string peer = item->PeerId(); + + std::map grp; mDataStore->retrieveGxsGrpMetaData(grp); @@ -1826,6 +2126,8 @@ void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) return; } + + bool RsGxsNetService::canSendGrpId(const std::string& sslId, RsGxsGrpMetaData& grpMeta, std::vector& toVet) { // first do the simple checks @@ -1881,10 +2183,31 @@ bool RsGxsNetService::canSendGrpId(const std::string& sslId, RsGxsGrpMetaData& g return true; } +bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncMsg *item) +{ + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId); + + if(cit != mServerMsgUpdateMap.end()) + { + const RsGxsServerMsgUpdateItem *msui = cit->second; + + if(item->updateTS >= msui->msgUpdateTS && item->updateTS != 0) + { +#ifdef NXS_NET_DEBUG + std::cerr << "RsGxsNetService::locked_CanReceiveUpdate(): Msgs up to date" << std::endl; +#endif + return false; + } + } + return true; +} void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsg* item) { RsStackMutex stack(mNxsMutex); + if(!locked_CanReceiveUpdate(item)) + return; + const std::string& peer = item->PeerId(); GxsMsgMetaResult metaResult; @@ -2066,3 +2389,40 @@ void RsGxsNetService::setSyncAge(uint32_t age) } +int RsGxsNetService::requestGrp(const std::list& grpId, const std::string& peerId) +{ + RsStackMutex stack(mNxsMutex); + mExplicitRequest[peerId].assign(grpId.begin(), grpId.end()); + return 1; +} + +void RsGxsNetService::processExplicitGroupRequests() +{ + RsStackMutex stack(mNxsMutex); + + std::map >::const_iterator cit = mExplicitRequest.begin(); + + for(; cit != mExplicitRequest.end(); cit++) + { + const std::string& peerId = cit->first; + const std::list& groupIdList = cit->second; + + std::list grpSyncItems; + std::list::const_iterator git = groupIdList.begin(); + uint32_t transN = locked_getTransactionId(); + for(; git != groupIdList.end(); git++) + { + RsNxsSyncGrpItem* item = new RsNxsSyncGrpItem(mServType); + item->grpId = *git; + item->PeerId(peerId); + item->flag = RsNxsSyncGrpItem::FLAG_REQUEST; + item->transactionNumber = transN; + grpSyncItems.push_back(item); + } + + if(!grpSyncItems.empty()) + locked_pushGrpTransactionFromList(grpSyncItems, peerId, transN); + } + + mExplicitRequest.clear(); +} diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 6c37d34a0..d72773f91 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -34,6 +34,7 @@ #include "rsnxsobserver.h" #include "pqi/p3linkmgr.h" #include "serialiser/rsnxsitems.h" +#include "serialiser/rsgxsupdateitems.h" #include "rsgxsnetutils.h" #include "pqi/p3cfgmgr.h" #include "rsgixs.h" @@ -73,7 +74,7 @@ public: * arrive */ RsGxsNetService(uint16_t servType, RsGeneralDataService* gds, RsNxsNetMgr* netMgr, - RsNxsObserver* nxsObs = NULL, RsGixsReputation* repuations = NULL, RsGcxs* circles = NULL); + RsNxsObserver* nxsObs = NULL, RsGixsReputation* repuations = NULL, RsGcxs* circles = NULL, bool grpAutoSync = true); virtual ~RsGxsNetService(); @@ -115,7 +116,7 @@ public: * @param msgId the messages to retrieve * @return request token to be redeemed */ - int requestMsg(const std::string& msgId, uint8_t hops){ return 0;} + int requestMsg(const RsGxsGrpMsgIdPair& msgId){ return 0;} /*! * Request for this group is sent through to peers on your network @@ -123,7 +124,7 @@ public: * @param enabled set to false to disable pause, and true otherwise * @return request token to be redeemed */ - int requestGrp(const std::list& grpId, uint8_t hops){ return 0;} + int requestGrp(const std::list& grpId, const std::string& peerId); /* p3Config methods */ @@ -322,6 +323,15 @@ private: bool locked_canReceive(const RsGxsGrpMetaData * const grpMeta, const std::string& peerId); + void processExplicitGroupRequests(); + + void locked_doMsgUpdateWork(const RsNxsTransac* nxsTrans, const std::string& grpId); + + void updateServerSyncTS(); + + bool locked_CanReceiveUpdate(const RsNxsSyncGrp* item); + bool locked_CanReceiveUpdate(const RsNxsSyncMsg* item); + private: typedef std::vector GrpFragments; @@ -418,10 +428,30 @@ private: RsGcxs* mCircles; RsGixsReputation* mReputations; + bool mGrpAutoSync; // need to be verfied std::vector mPendingResp; std::vector mPendingCircleVets; + + std::map > mExplicitRequest; + + // nxs sync optimisation + // can pull dynamically the latest timestamp for each message + +public: + + typedef std::map ClientMsgMap; + typedef std::map ServerMsgMap; + typedef std::map ClientGrpMap; + +private: + + ClientMsgMap mClientMsgUpdateMap; + ServerMsgMap mServerMsgUpdateMap; + ClientGrpMap mClientGrpUpdateMap; + + RsGxsServerGrpUpdateItem* mGrpServerUpdateItem; }; #endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/gxs/rsgxsnetutils.cc b/libretroshare/src/gxs/rsgxsnetutils.cc index f61ee033c..e5fb94447 100644 --- a/libretroshare/src/gxs/rsgxsnetutils.cc +++ b/libretroshare/src/gxs/rsgxsnetutils.cc @@ -47,14 +47,16 @@ bool AuthorPending::expired() const } bool AuthorPending::getAuthorRep(GixsReputation& rep, - const std::string& authorId) + const std::string& authorId, const std::string& peerId) { if(mRep->haveReputation(authorId)) { return mRep->getReputation(authorId, rep); } - mRep->loadReputation(authorId); + std::list peers; + peers.push_back(peerId); + mRep->loadReputation(authorId, peers); return false; } @@ -94,7 +96,7 @@ bool MsgRespPending::accepted() if(!entry.mPassedVetting) { GixsReputation rep; - if(getAuthorRep(rep, entry.mAuthorId)) + if(getAuthorRep(rep, entry.mAuthorId, mPeerId)) { if(rep.score > mCutOff) { @@ -129,7 +131,7 @@ bool GrpRespPending::accepted() { GixsReputation rep; - if(getAuthorRep(rep, entry.mAuthorId)) + if(getAuthorRep(rep, entry.mAuthorId, mPeerId)) { if(rep.score > mCutOff) { diff --git a/libretroshare/src/gxs/rsgxsnetutils.h b/libretroshare/src/gxs/rsgxsnetutils.h index 1bd70b9b9..b93283aac 100644 --- a/libretroshare/src/gxs/rsgxsnetutils.h +++ b/libretroshare/src/gxs/rsgxsnetutils.h @@ -141,7 +141,7 @@ protected: * @param authorId reputation to get * @return true if successfully retrieve repution */ - bool getAuthorRep(GixsReputation& rep, const std::string& authorId); + bool getAuthorRep(GixsReputation& rep, const std::string& authorId, const std::string& peerId); private: diff --git a/libretroshare/src/gxs/rsgxsutil.h b/libretroshare/src/gxs/rsgxsutil.h index b15e3479f..2828ea637 100644 --- a/libretroshare/src/gxs/rsgxsutil.h +++ b/libretroshare/src/gxs/rsgxsutil.h @@ -129,4 +129,24 @@ private: }; +class GroupUpdate +{ +public: + GroupUpdate() : oldGrpMeta(NULL), newGrp(NULL), validUpdate(false) + {} + RsGxsGrpMetaData* oldGrpMeta; + RsNxsGrp* newGrp; + bool validUpdate; +}; + +class GroupUpdatePublish +{ +public: + GroupUpdatePublish(RsGxsGrpItem* item, RsGxsGroupUpdateMeta updateMeta, uint32_t token) + : grpItem(item), mUpdateMeta(updateMeta), mToken(token) {} + RsGxsGrpItem* grpItem; + RsGxsGroupUpdateMeta mUpdateMeta; + uint32_t mToken; +}; + #endif /* GXSUTIL_H_ */ diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 0c201a737..e909994a2 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -68,21 +68,6 @@ public: */ virtual void setSyncAge(uint32_t age) = 0; - /*! - * Explicitly requests all the groups contained by a peer - * Circumvents polling of peers for message - * @param peerId id of peer - */ - virtual void requestGroupsOfPeer(const std::string& peerId) = 0; - - /*! - * get messages of a peer for a given group id, this circumvents the normal - * polling of peers for messages of given group id - * @param peerId Id of peer - * @param grpId id of group to request messages for - */ - virtual void requestMessagesOfPeer(const std::string& peerId, const std::string& grpId) = 0; - /*! * Initiates a search through the network * This returns messages which contains the search terms set in RsGxsSearch @@ -116,7 +101,7 @@ public: * @param msgId the messages to retrieve * @return request token to be redeemed */ - virtual int requestMsg(const std::string& msgId, uint8_t hops) = 0; + virtual int requestMsg(const RsGxsGrpMsgIdPair& msgId) = 0; /*! * Request for this group is sent through to peers on your network @@ -124,7 +109,7 @@ public: * @param enabled set to false to disable pause, and true otherwise * @return request token to be redeemed */ - virtual int requestGrp(const std::list& grpId, uint8_t hops) = 0; + virtual int requestGrp(const std::list& grpId, const std::string& peerId) = 0; }; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 72db6cf52..6185eca0e 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -3,18 +3,13 @@ CONFIG += staticlib bitdht CONFIG -= qt TARGET = retroshare -CONFIG += test_voip #GXS Stuff. # This should be disabled for releases until further notice. -#CONFIG += gxs debug +CONFIG += gxs debug #CONFIG += grouter #CONFIG += dsdv -# Beware: All data of the stripped services are lost -DEFINES *= PQI_DISABLE_TUNNEL -#ENABLE_CACHE_OPT - profiling { QMAKE_CXXFLAGS -= -fomit-frame-pointer QMAKE_CXXFLAGS *= -pg -g -fno-omit-frame-pointer @@ -107,13 +102,9 @@ SOURCES += tcponudp/udppeer.cc \ -PUBLIC_HEADERS = retroshare/rsblogs.h \ - retroshare/rschannels.h \ - retroshare/rsdisc.h \ - retroshare/rsdistrib.h \ +PUBLIC_HEADERS = retroshare/rsdisc.h \ retroshare/rsexpr.h \ retroshare/rsfiles.h \ - retroshare/rsforums.h \ retroshare/rshistory.h \ retroshare/rsiface.h \ retroshare/rsinit.h \ @@ -127,17 +118,16 @@ PUBLIC_HEADERS = retroshare/rsblogs.h \ retroshare/rsturtle.h \ retroshare/rstypes.h \ retroshare/rsdht.h \ + retroshare/rsrtt.h \ retroshare/rsconfig.h + HEADERS += plugins/pluginmanager.h \ plugins/dlfcn_win32.h \ serialiser/rspluginitems.h HEADERS += $$PUBLIC_HEADERS -# public headers to be... -HEADERS += retroshare/rsgame.h \ - retroshare/rsphoto.h ################################# Linux ########################################## linux-* { @@ -338,8 +328,6 @@ HEADERS += dbase/cachestrapper.h \ dbase/findex.h \ dbase/fistore.h -#HEADERS += dht/p3bitdht.h \ - HEADERS += ft/ftchunkmap.h \ ft/ftcontroller.h \ ft/ftdata.h \ @@ -364,7 +352,6 @@ HEADERS += pqi/authssl.h \ pqi/p3peermgr.h \ pqi/p3linkmgr.h \ pqi/p3netmgr.h \ - pqi/p3dhtmgr.h \ pqi/p3notify.h \ pqi/p3upnpmgr.h \ pqi/pqiqos.h \ @@ -389,31 +376,29 @@ HEADERS += pqi/authssl.h \ pqi/pqissl.h \ pqi/pqissllistener.h \ pqi/pqisslpersongrp.h \ - pqi/pqissltunnel.h \ pqi/pqissludp.h \ + pqi/pqisslproxy.h \ pqi/pqistore.h \ pqi/pqistreamer.h \ + pqi/pqithreadstreamer.h \ pqi/pqiqosstreamer.h \ pqi/sslfns.h \ pqi/pqinetstatebox.h -HEADERS += rsserver/p3discovery.h \ - rsserver/p3face.h \ +# pqi/p3dhtmgr.h \ + +HEADERS += rsserver/p3face.h \ rsserver/p3history.h \ rsserver/p3msgs.h \ rsserver/p3peers.h \ rsserver/p3status.h \ + rsserver/rsaccounts.h \ rsserver/p3serverconfig.h -HEADERS += serialiser/rsbaseitems.h \ - serialiser/rsbaseserial.h \ - serialiser/rsblogitems.h \ - serialiser/rschannelitems.h \ +HEADERS += serialiser/rsbaseserial.h \ + serialiser/rsfiletransferitems.h \ + serialiser/rsserviceserialiser.h \ serialiser/rsconfigitems.h \ - serialiser/rsdiscitems.h \ - serialiser/rsdistribitems.h \ - serialiser/rsforumitems.h \ - serialiser/rsgameitems.h \ serialiser/rshistoryitems.h \ serialiser/rsmsgitems.h \ serialiser/rsserial.h \ @@ -429,32 +414,29 @@ HEADERS += serialiser/rsbaseitems.h \ serialiser/rstlvbanlist.h \ serialiser/rsbanlistitems.h \ serialiser/rsbwctrlitems.h \ - serialiser/rstunnelitems.h + serialiser/rsdiscovery2items.h \ + serialiser/rsheartbeatitems.h \ + serialiser/rsrttitems.h \ + serialiser/rsgxsrecognitems.h \ + serialiser/rsgxsupdateitems.h -HEADERS += services/p3channels.h \ - services/p3chatservice.h \ - services/p3disc.h \ - services/p3forums.h \ - services/p3gamelauncher.h \ - services/p3gameservice.h \ +HEADERS += services/p3chatservice.h \ services/p3msgservice.h \ services/p3service.h \ services/p3statusservice.h \ services/p3banlist.h \ services/p3bwctrl.h \ - services/p3tunnel.h - -HEADERS += distrib/p3distrib.h \ - distrib/p3distribsecurity.h -# services/p3blogs.h \ + services/p3discovery2.h \ + services/p3heartbeat.h \ + services/p3rtt.h \ HEADERS += turtle/p3turtle.h \ turtle/rsturtleitem.h \ turtle/turtletypes.h - HEADERS += util/folderiterator.h \ util/rsdebug.h \ + util/rscompress.h \ util/smallobject.h \ util/rsdir.h \ util/rsdiscspace.h \ @@ -471,6 +453,7 @@ HEADERS += util/folderiterator.h \ util/pugiconfig.h \ util/rsmemcache.h \ util/rstickevent.h \ + util/rsrecogn.h \ SOURCES += dbase/cachestrapper.cc \ dbase/fimonitor.cc \ @@ -500,7 +483,6 @@ SOURCES += pqi/authgpg.cc \ pqi/p3peermgr.cc \ pqi/p3linkmgr.cc \ pqi/p3netmgr.cc \ - pqi/p3dhtmgr.cc \ pqi/p3notify.cc \ pqi/pqiqos.cc \ pqi/pqiarchive.cc \ @@ -518,16 +500,18 @@ SOURCES += pqi/authgpg.cc \ pqi/pqissl.cc \ pqi/pqissllistener.cc \ pqi/pqisslpersongrp.cc \ - pqi/pqissltunnel.cc \ pqi/pqissludp.cc \ + pqi/pqisslproxy.cc \ pqi/pqistore.cc \ pqi/pqistreamer.cc \ + pqi/pqithreadstreamer.cc \ pqi/pqiqosstreamer.cc \ pqi/sslfns.cc \ pqi/pqinetstatebox.cc -SOURCES += rsserver/p3discovery.cc \ - rsserver/p3face-config.cc \ +# pqi/p3dhtmgr.cc \ + +SOURCES += rsserver/p3face-config.cc \ rsserver/p3face-msgs.cc \ rsserver/p3face-server.cc \ rsserver/p3history.cc \ @@ -535,6 +519,7 @@ SOURCES += rsserver/p3discovery.cc \ rsserver/p3peers.cc \ rsserver/p3status.cc \ rsserver/rsinit.cc \ + rsserver/rsaccounts.cc \ rsserver/rsloginhandler.cc \ rsserver/rstypes.cc \ rsserver/p3serverconfig.cc @@ -543,15 +528,10 @@ SOURCES += plugins/pluginmanager.cc \ plugins/dlfcn_win32.cc \ serialiser/rspluginitems.cc -SOURCES += serialiser/rsbaseitems.cc \ - serialiser/rsbaseserial.cc \ - serialiser/rsblogitems.cc \ - serialiser/rschannelitems.cc \ +SOURCES += serialiser/rsbaseserial.cc \ + serialiser/rsfiletransferitems.cc \ + serialiser/rsserviceserialiser.cc \ serialiser/rsconfigitems.cc \ - serialiser/rsdiscitems.cc \ - serialiser/rsdistribitems.cc \ - serialiser/rsforumitems.cc \ - serialiser/rsgameitems.cc \ serialiser/rshistoryitems.cc \ serialiser/rsmsgitems.cc \ serialiser/rsserial.cc \ @@ -567,24 +547,22 @@ SOURCES += serialiser/rsbaseitems.cc \ serialiser/rstlvbanlist.cc \ serialiser/rsbanlistitems.cc \ serialiser/rsbwctrlitems.cc \ - serialiser/rstunnelitems.cc + serialiser/rsdiscovery2items.cc \ + serialiser/rsheartbeatitems.cc \ + serialiser/rsrttitems.cc \ + serialiser/rsgxsrecognitems.cc \ + serialiser/rsgxsupdateitems.cc -SOURCES += services/p3channels.cc \ - services/p3chatservice.cc \ - services/p3disc.cc \ - services/p3forums.cc \ - services/p3gamelauncher.cc \ +SOURCES += services/p3chatservice.cc \ services/p3msgservice.cc \ services/p3service.cc \ services/p3statusservice.cc \ services/p3banlist.cc \ services/p3bwctrl.cc \ + services/p3discovery2.cc \ + services/p3heartbeat.cc \ + services/p3rtt.cc \ -# removed because getPeer() doesn t exist services/p3tunnel.cc - -SOURCES += distrib/p3distrib.cc \ - distrib/p3distribsecurity.cc - SOURCES += turtle/p3turtle.cc \ turtle/rsturtleitem.cc # turtle/turtlerouting.cc \ @@ -594,10 +572,12 @@ SOURCES += turtle/p3turtle.cc \ SOURCES += util/folderiterator.cc \ util/rsdebug.cc \ + util/rscompress.cc \ util/smallobject.cc \ util/rsdir.cc \ util/rsdiscspace.cc \ util/rsnet.cc \ + util/rsnet_ss.cc \ util/extaddrfinder.cc \ util/dnsresolver.cc \ util/rsprint.cc \ @@ -608,6 +588,7 @@ SOURCES += util/folderiterator.cc \ util/rsaes.cc \ util/rsrandom.cc \ util/rstickevent.cc \ + util/rsrecogn.cc \ upnp_miniupnpc { @@ -752,11 +733,13 @@ gxs { serialiser/rswireitems.cc \ # Posted Service - HEADERS += services/p3posted.h \ + HEADERS += services/p3postbase.h \ + services/p3posted.h \ retroshare/rsposted.h \ serialiser/rsposteditems.h - SOURCES += services/p3posted.cc \ + SOURCES += services/p3postbase.cc \ + services/p3posted.cc \ serialiser/rsposteditems.cc #Photo Service @@ -809,12 +792,3 @@ test_bitdht { - -use_blogs { - - HEADERS += services/p3blogs.h - SOURCES += services/p3blogs.cc - - DEFINES *= RS_USE_BLOGS -} - diff --git a/libretroshare/src/pgp/pgphandler.cc b/libretroshare/src/pgp/pgphandler.cc index 035c70168..cf9e16ec2 100644 --- a/libretroshare/src/pgp/pgphandler.cc +++ b/libretroshare/src/pgp/pgphandler.cc @@ -1808,8 +1808,8 @@ bool PGPHandler::locked_syncPublicKeyring() bool PGPHandler::locked_syncTrustDatabase() { struct stat64 buf ; - std::wstring wfullname; #ifdef WINDOWS_SYS + std::wstring wfullname; librs::util::ConvertUtf8ToUtf16(_trustdb_path, wfullname); if(-1 == _wstati64(wfullname.c_str(), &buf)) #else diff --git a/libretroshare/src/pgp/rscertificate.cc b/libretroshare/src/pgp/rscertificate.cc index fb18791dc..20617d184 100644 --- a/libretroshare/src/pgp/rscertificate.cc +++ b/libretroshare/src/pgp/rscertificate.cc @@ -7,10 +7,9 @@ #include #include #include "rscertificate.h" +#include "util/rsstring.h" -//#define DEBUG_RSCERTIFICATE - -//#define V_06_USE_CHECKSUM +#define DEBUG_RSCERTIFICATE static const std::string PGP_CERTIFICATE_START ( "-----BEGIN PGP PUBLIC KEY BLOCK-----" ); static const std::string PGP_CERTIFICATE_END ( "-----END PGP PUBLIC KEY BLOCK-----" ); @@ -18,6 +17,7 @@ static const std::string EXTERNAL_IP_BEGIN_SECTION ( "--EXT--" ); static const std::string LOCAL_IP_BEGIN_SECTION ( "--LOCAL--" ); static const std::string SSLID_BEGIN_SECTION ( "--SSLID--" ); static const std::string LOCATION_BEGIN_SECTION ( "--LOCATION--" ); +static const std::string HIDDEN_NODE_BEGIN_SECTION ( "--HIDDEN--" ); static const uint8_t CERTIFICATE_PTAG_PGP_SECTION = 0x01 ; static const uint8_t CERTIFICATE_PTAG_EXTIPANDPORT_SECTION = 0x02 ; @@ -26,6 +26,10 @@ static const uint8_t CERTIFICATE_PTAG_DNS_SECTION = 0x04 ; static const uint8_t CERTIFICATE_PTAG_SSLID_SECTION = 0x05 ; static const uint8_t CERTIFICATE_PTAG_NAME_SECTION = 0x06 ; static const uint8_t CERTIFICATE_PTAG_CHECKSUM_SECTION = 0x07 ; +static const uint8_t CERTIFICATE_PTAG_HIDDENNODE_SECTION = 0x08 ; +static const uint8_t CERTIFICATE_PTAG_VERSION_SECTION = 0x09 ; + +static const uint8_t CERTIFICATE_VERSION_06 = 0x06 ; static bool is_acceptable_radix64Char(char c) { @@ -73,17 +77,26 @@ std::string RsCertificate::toStdString() const size_t p = 0 ; unsigned char *buf = new unsigned char[BS] ; - addPacket( CERTIFICATE_PTAG_PGP_SECTION , binary_pgp_key , binary_pgp_key_size , buf, p, BS ) ; + addPacket( CERTIFICATE_PTAG_VERSION_SECTION, &CERTIFICATE_VERSION_06 , 1 , buf, p, BS ) ; + addPacket( CERTIFICATE_PTAG_PGP_SECTION , binary_pgp_key , binary_pgp_key_size , buf, p, BS ) ; if(!only_pgp) { - addPacket( CERTIFICATE_PTAG_EXTIPANDPORT_SECTION, ipv4_external_ip_and_port , 6 , buf, p, BS ) ; - addPacket( CERTIFICATE_PTAG_LOCIPANDPORT_SECTION, ipv4_internal_ip_and_port , 6 , buf, p, BS ) ; - addPacket( CERTIFICATE_PTAG_DNS_SECTION , (unsigned char *)dns_name.c_str() , dns_name.length() , buf, p, BS ) ; + if (hidden_node) + { + addPacket( CERTIFICATE_PTAG_HIDDENNODE_SECTION, (unsigned char *)hidden_node_address.c_str(), hidden_node_address.length() , buf, p, BS ) ; + } + else + { + addPacket( CERTIFICATE_PTAG_EXTIPANDPORT_SECTION, ipv4_external_ip_and_port , 6 , buf, p, BS ) ; + addPacket( CERTIFICATE_PTAG_LOCIPANDPORT_SECTION, ipv4_internal_ip_and_port , 6 , buf, p, BS ) ; + addPacket( CERTIFICATE_PTAG_DNS_SECTION , (unsigned char *)dns_name.c_str() , dns_name.length() , buf, p, BS ) ; + } + addPacket( CERTIFICATE_PTAG_NAME_SECTION , (unsigned char *)location_name.c_str() ,location_name.length() , buf, p, BS ) ; addPacket( CERTIFICATE_PTAG_SSLID_SECTION , location_id.toByteArray() ,location_id.SIZE_IN_BYTES, buf, p, BS ) ; } -#ifdef V_06_USE_CHECKSUM + uint32_t computed_crc = PGPKeyManagement::compute24bitsCRC(buf,p) ; // handle endian issues. @@ -93,7 +106,7 @@ std::string RsCertificate::toStdString() const mem[2] = (computed_crc >> 16) & 0xff ; addPacket( CERTIFICATE_PTAG_CHECKSUM_SECTION,mem,3,buf,p,BS) ; -#endif + std::string out_string ; Radix64::encode((char *)buf, p, out_string) ; @@ -124,7 +137,7 @@ RsCertificate::RsCertificate(const std::string& str) uint32_t err_code ; binary_pgp_key = NULL ; - if(!initFromString(str,err_code) && !initFromString_oldFormat(str,err_code)) + if(!initFromString(str,err_code)) throw err_code ; } @@ -144,14 +157,52 @@ RsCertificate::RsCertificate(const RsPeerDetails& Detail, const unsigned char *b location_id = SSLIdType( Detail.id ) ; location_name = Detail.location ; - scan_ip(Detail.localAddr,Detail.localPort,ipv4_internal_ip_and_port) ; - scan_ip(Detail.extAddr,Detail.extPort,ipv4_external_ip_and_port) ; + if (Detail.isHiddenNode) + { + hidden_node = true; + hidden_node_address = Detail.hiddenNodeAddress; + rs_sprintf_append(hidden_node_address, ":%u", Detail.hiddenNodePort); - dns_name = Detail.dyndns ; + memset(ipv4_internal_ip_and_port,0,6) ; + memset(ipv4_external_ip_and_port,0,6) ; + dns_name = "" ; + } + else + { + hidden_node = false; + hidden_node_address = ""; + + try + { + scan_ip(Detail.localAddr,Detail.localPort,ipv4_internal_ip_and_port) ; + } + catch(...) + { + std::cerr << "RsCertificate::Invalid LocalAddress"; + std::cerr << std::endl; + memset(ipv4_internal_ip_and_port,0,6) ; + } + + + try + { + scan_ip(Detail.extAddr,Detail.extPort,ipv4_external_ip_and_port) ; + } + catch(...) + { + std::cerr << "RsCertificate::Invalid ExternalAddress"; + std::cerr << std::endl; + memset(ipv4_external_ip_and_port,0,6) ; + } + + dns_name = Detail.dyndns ; + } } else { only_pgp = true ; + hidden_node = false; + hidden_node_address = ""; location_id = SSLIdType() ; location_name = "" ; memset(ipv4_internal_ip_and_port,0,6) ; @@ -208,6 +259,7 @@ bool RsCertificate::initFromString(const std::string& instr,uint32_t& err_code) unsigned char *buf = (unsigned char *)bf ; size_t total_s = 0 ; only_pgp = true ; + uint8_t certificate_version = 0x00 ; while(total_s < size) { @@ -230,6 +282,11 @@ bool RsCertificate::initFromString(const std::string& instr,uint32_t& err_code) #endif switch(ptag) { + case CERTIFICATE_PTAG_VERSION_SECTION: certificate_version = buf[0] ; + buf = &buf[s] ; + break ; + + case CERTIFICATE_PTAG_PGP_SECTION: binary_pgp_key = new unsigned char[s] ; memcpy(binary_pgp_key,buf,s) ; binary_pgp_key_size = s ; @@ -256,8 +313,16 @@ bool RsCertificate::initFromString(const std::string& instr,uint32_t& err_code) buf = &buf[s] ; break ; + case CERTIFICATE_PTAG_HIDDENNODE_SECTION: + hidden_node_address = std::string((char *)buf,s); + hidden_node = true; + buf = &buf[s]; + + break ; + case CERTIFICATE_PTAG_LOCIPANDPORT_SECTION: if(s != 6) + { err_code = CERTIFICATE_PARSING_ERROR_INVALID_LOCAL_IP; return false ; @@ -302,12 +367,20 @@ bool RsCertificate::initFromString(const std::string& instr,uint32_t& err_code) total_s += s ; } -#ifdef V_06_USE_CHECKSUM + if(!checksum_check_passed) { err_code = CERTIFICATE_PARSING_ERROR_MISSING_CHECKSUM ; return false ; } + + if(certificate_version != CERTIFICATE_VERSION_06) + { + err_code = CERTIFICATE_PARSING_ERROR_WRONG_VERSION ; + return false ; + } +#ifdef DEBUG_RSCERTIFICATE + std::cerr << "Certificate is version " << (int)certificate_version << std::endl; #endif if(total_s != size) @@ -326,6 +399,17 @@ bool RsCertificate::initFromString(const std::string& instr,uint32_t& err_code) } } +std::string RsCertificate::hidden_node_string() const +{ + if ((!only_pgp) && (hidden_node)) + { + return hidden_node_address; + } + + std::string empty; + return empty; +} + std::string RsCertificate::sslid_string() const { if (only_pgp) @@ -364,12 +448,6 @@ unsigned short RsCertificate::loc_port_us() const bool RsCertificate::cleanCertificate(const std::string& input,std::string& output,Format& format,int& error_code) { - if(cleanCertificate_oldFormat(input,output,error_code)) - { - format = RS_CERTIFICATE_OLD_FORMAT ; - return true ; - } - if(cleanCertificate(input,output,error_code)) { format = RS_CERTIFICATE_RADIX ; @@ -423,557 +501,7 @@ bool RsCertificate::cleanCertificate(const std::string& instr,std::string& str,i return true ; } -// All the code below should be removed when in 0.6. Certificates will only use the new format. -// -bool RsCertificate::cleanCertificate_oldFormat(const std::string& certstr,std::string& cleanCertificate,int& error_code) -{ - error_code = RS_PEER_CERT_CLEANING_CODE_UNKOWN_ERROR ; // default - const std::string& badCertificate(certstr) ; - - std::string pgpend("-----END PGP PUBLIC KEY BLOCK-----"); - - size_t pos = certstr.find(pgpend); - std::string peer_info ; - std::string cert ; - - if (pos != std::string::npos) - { - pos += pgpend.length(); - cert = certstr.substr(0, pos); - if (pos + 1 < certstr.length()) - peer_info = certstr.substr(pos + 1); - } - else - { - error_code = RS_PEER_CERT_CLEANING_CODE_NO_END_TAG ; - return false ; - } - - if(cert.empty()) - return false ; - - /* - Buffer for storing the cleaned certificate. In certain cases the - cleanCertificate can be larger than the badCertificate - */ - cleanCertificate = ""; - //The entire certificate begin tag - const char * beginCertTag="-----BEGIN"; - //The entire certificate end tag - const char * endCertTag="-----END"; - //Tag containing dots. The common part of both start and end tags - const char * commonTag="-----"; - //Only BEGIN part of the begin tag - const char * beginTag="BEGIN"; - //Only END part of the end tag - const char * endTag="END"; - //The start index of the ----- part of the certificate begin tag - size_t beginCertStartIdx1=0; - //The start index of the BEGIN part of the certificate begin tag - size_t beginCertStartIdx2=0; - //The start index of the end part(-----) of the certificate begin tag. The begin tag ends with -----. Example -----BEGIN XPGP CERTIFICATE----- - size_t beginCertEndIdx=0; - //The start index of the ----- part of the certificate end tag - size_t endCertStartIdx1=0; - //The start index of the END part of the certificate end tag - size_t endCertStartIdx2=0; - //The start index of the end part(-----) of the certificate end tag. The begin tag ends with -----. Example -----BEGIN XPGP CERTIFICATE----- - size_t endCertEndIdx=0; - //The length of the bad certificate. - size_t lengthOfCert=certstr.length(); - //The current index value in the bad certificate - size_t currBadCertIdx=0; - //Temporary index value - size_t tmpIdx=0; - //Boolean flag showing if the begin tag or the end tag has been found - bool found=false; - /* - Calculating the value of the beginCertStartIdx1 and beginCertStartIdx2. Here - we first locate the occurance of ----- and then the location of BEGIN. Next - we check if there are any non space or non new-line characters between their - occureance. If there are any other characters between the two(----- and - BEGIN), other than space and new line then it means that it is the - certificate begin tag. Here we take care of the fact that we may have - introduced some spaces and newlines in the begin tag by mistake. This takes - care of the spaces and newlines between ----- and BEGIN. - */ - - while(found==false && (beginCertStartIdx1=certstr.find(commonTag,tmpIdx))!=std::string::npos) - { - beginCertStartIdx2=certstr.find(beginTag,beginCertStartIdx1+strlen(commonTag)); - tmpIdx=beginCertStartIdx1+strlen(commonTag); - if(beginCertStartIdx2!=std::string::npos) - { - found=true; - for(size_t i=beginCertStartIdx1+strlen(commonTag);i tag"< tag"< tag"<=lengthOfCert) - { - std::cerr<<"Certificate corrupted beyond repair: No <------END > tag"< header; - header.push_back("Version"); - header.push_back("Comment"); - header.push_back("MessageID"); - header.push_back("Hash"); - header.push_back("Charset"); - - for (std::list::iterator headerIt = header.begin (); headerIt != header.end(); headerIt++) - { - if (badCertificate.substr(currBadCertIdx, (*headerIt).length()) == *headerIt) - { - cleanCertificate += badCertificate.substr(currBadCertIdx, (*headerIt).length()); - currBadCertIdx += (*headerIt).length(); - while(currBadCertIdx=endCertStartIdx1) - { - std::cerr<<"Certificate corrupted beyond repair: No checksum, or no newline after first tag"<