diff --git a/libretroshare/src/dbase/cachestrapper.cc b/libretroshare/src/dbase/cachestrapper.cc deleted file mode 100644 index 4ecdedf64..000000000 --- a/libretroshare/src/dbase/cachestrapper.cc +++ /dev/null @@ -1,1323 +0,0 @@ -/* - * RetroShare FileCache Module: cachestrapper.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include "dbase/cachestrapper.h" -#include "serialiser/rsserviceids.h" -#include "serialiser/rsconfigitems.h" -#include "pqi/p3servicecontrol.h" -#include "pqi/p3peermgr.h" -#include "util/rsdir.h" - -#include -#include - -/**** -* #define CS_DEBUG 1 -***/ - -bool operator<(const CacheId &a, const CacheId &b) -{ - if (a.type == b.type) - return (a.subid < b.subid); - return (a.type < b.type); -} - -bool operator<(const CachePair &a, const CachePair &b) -{ - return (a.id < b.id); -} - - -std::ostream &operator<<(std::ostream &out, const RsCacheData &d) -{ - out << "[ p: " << d.pid << " id: <" << d.cid.type << "," << d.cid.subid; - out << "> #" << d.hash << " size: " << d.size; - out << " \"" << d.name << "\"@\"" << d.path; - out << "\" ]"; - return out; -} - -/********************************* Cache Store / Source ************************** - * This is a generic interface which interacts with the FileTransfer Unit - * to collect Data to be Cached. - * - ********************************* Cache Store / Source *************************/ - -CacheSource::CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir) - :cacheType(t), multiCache(m), mStrapper(cs), cacheDir(cachedir), cMutex("CacheSource") - { - return; - } - - /* Mutex Stuff -> to be done... */ -void CacheSource::lockData() const -{ -#ifdef CS_DEBUG - std::cerr << "CacheSource::lockData()" << std::endl; -#endif - cMutex.lock(); -} - -void CacheSource::unlockData() const -{ -#ifdef CS_DEBUG - std::cerr << "CacheSource::unlockData()" << std::endl; -#endif - cMutex.unlock(); -} - - /* to be overloaded for inherited Classes */ -bool CacheSource::loadLocalCache(const RsCacheData &data) -{ - return refreshCache(data); -} - - /* control Caches available */ -bool CacheSource::refreshCache(const RsCacheData &data,const std::set& destination_peers) -{ - bool ret = false; - { - RsStackMutex mtx(cMutex); /* LOCK MUTEX */ - - if (data.cid.type == getCacheType()) - { - int subid = 0; - if (isMultiCache()) - { - subid = data.cid.subid; - } - - /* Backup the old Caches */ - CacheSet::const_iterator it; - if (caches.end() != (it = caches.find(subid))) - { - mOldCaches[it->second.hash] = it->second; - } - - /* store new cache */ - caches[subid] = data; - ret = true; - } - } - // Strip down destination peers to eliminate peers that are not allowed to receive cache items. - - if (mStrapper) /* allow testing without full feedback */ - { - std::set allowed_dest_peers ; - - for(std::set::const_iterator it(destination_peers.begin());it!=destination_peers.end();++it) - if(isPeerAcceptedAsCacheReceiver(*it)) - allowed_dest_peers.insert(*it) ; - - mStrapper->refreshCache(data,allowed_dest_peers); - } - - return ret; -} -bool CacheSource::refreshCache(const RsCacheData &data) -{ - bool ret = false; - { - RsStackMutex mtx(cMutex); /* LOCK MUTEX */ - - if (data.cid.type == getCacheType()) - { - int subid = 0; - if (isMultiCache()) - { - subid = data.cid.subid; - } - - /* Backup the old Caches */ - CacheSet::const_iterator it; - if (caches.end() != (it = caches.find(subid))) - { - mOldCaches[it->second.hash] = it->second; - } - - /* store new cache */ - caches[subid] = data; - ret = true; - } - } - // Strip down destination peers to eliminate peers that are not allowed to receive cache items. - - std::list ids; - rsPeers->getOnlineList(ids); - - if (mStrapper) /* allow testing without full feedback */ - { - std::set allowed_dest_peers ; - - for(std::list::const_iterator it(ids.begin());it!=ids.end();++it) - if(isPeerAcceptedAsCacheReceiver(*it)) - allowed_dest_peers.insert(*it) ; - - mStrapper->refreshCache(data,allowed_dest_peers); - } - - return ret; - -} - -// bool CacheSource::refreshCache(const RsCacheData &data) -// { -// bool ret = false; -// { -// RsStackMutex mtx(cMutex); /* LOCK MUTEX */ -// -// if (data.cid.type == getCacheType()) -// { -// int subid = 0; -// if (isMultiCache()) -// { -// subid = data.cid.subid; -// } -// -// /* Backup the old Caches */ -// CacheSet::const_iterator it; -// if (caches.end() != (it = caches.find(subid))) -// { -// mOldCaches[it->second.hash] = it->second; -// } -// -// /* store new cache */ -// caches[subid] = data; -// ret = true; -// } -// } -// -// if (mStrapper) /* allow testing without full feedback */ -// mStrapper->refreshCache(data); -// -// return ret; -// } -bool CacheSource::clearCache(CacheId id) -{ - lockData(); /* LOCK MUTEX */ - - bool ret = false; - if (id.type == getCacheType()) - { - CacheSet::iterator it; - if (caches.end() != (it = caches.find(id.subid))) - { - /* Backup the old Caches */ - mOldCaches[it->second.hash] = it->second; - caches.erase(it); - ret = true; - } - } - - unlockData(); /* UNLOCK MUTEX */ - return ret; -} - -//bool CacheSource::cachesAvailable(const RsPeerId& pid, std::map &ids) -//{ -// if(!isPeerAcceptedAsCacheReceiver(pid)) -// return false ; -// -// lockData(); /* LOCK MUTEX */ -// -// /* can overwrite for more control! */ -// CacheSet::iterator it; -// for(it = caches.begin(); it != caches.end(); ++it) -// { -// ids[(it->second).cid] = it->second; -// } -// bool ret = (caches.size() > 0); -// -// unlockData(); /* UNLOCK MUTEX */ -// return ret; -// -//} - - -bool CacheSource::findCache(const RsFileHash &hash, RsCacheData &data) const -{ - lockData(); /* LOCK MUTEX */ - - bool found = false; - CacheSet::const_iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - if (hash == (it->second).hash) - { - data = it->second; - found = true; - break; - } - } - - if (!found) - { - std::map::const_iterator oit; - oit = mOldCaches.find(hash); - if (oit != mOldCaches.end()) - { - data = oit->second; - found = true; - } - } - - unlockData(); /* UNLOCK MUTEX */ - - return found; -} - - -void CacheSource::listCaches(std::ostream &out) -{ - lockData(); /* LOCK MUTEX */ - - /* can overwrite for more control! */ - CacheSet::iterator it; - out << "CacheSource::listCaches() [" << getCacheType(); - out << "] Total: " << caches.size() << std::endl; - int i; - for(i = 0, it = caches.begin(); it != caches.end(); ++it, ++i) - { - out << "\tC[" << i << "] : " << it->second << std::endl; - } - - unlockData(); /* UNLOCK MUTEX */ - return; -} - - -CacheStore::CacheStore(uint16_t t, bool m, - CacheStrapper *cs, CacheTransfer *cft, std::string cachedir) - :cacheType(t), multiCache(m), mStrapper(cs), - cacheTransfer(cft), cacheDir(cachedir), cMutex("CacheStore") - { - /* not much */ - return; - } - - /* Mutex Stuff -> to be done... */ -void CacheStore::lockData() const -{ -#ifdef CS_DEBUG -// std::cerr << "CacheStore::lockData()" << std::endl; -#endif - cMutex.lock(); -} - -void CacheStore::unlockData() const -{ -#ifdef CS_DEBUG -// std::cerr << "CacheStore::unlockData()" << std::endl; -#endif - cMutex.unlock(); -} - -void CacheStore::listCaches(std::ostream &out) -{ - lockData(); /* LOCK MUTEX */ - - /* can overwrite for more control! */ - std::map::iterator pit; - out << "CacheStore::listCaches() [" << getCacheType(); - out << "] Total People: " << caches.size(); - out << std::endl; - for(pit = caches.begin(); pit != caches.end(); ++pit) - { - CacheSet::iterator it; - out << "\tTotal for [" << pit->first << "] : " << (pit->second).size(); - out << std::endl; - for(it = (pit->second).begin(); it != (pit->second).end(); ++it) - { - out << "\t\t" << it->second; - out << std::endl; - } - } - - unlockData(); /* UNLOCK MUTEX */ - return; -} - - - /* look for stored data. using pic/cid in RsCacheData - */ -bool CacheStore::getStoredCache(RsCacheData &data) -{ - lockData(); /* LOCK MUTEX */ - - bool ok = locked_getStoredCache(data); - - unlockData(); /* UNLOCK MUTEX */ - return ok; -} - - -bool CacheStore::locked_getStoredCache(RsCacheData &data) -{ - if (data.cid.type != getCacheType()) - { - return false; - } - - std::map::iterator pit; - if (caches.end() == (pit = caches.find(data.pid))) - { - return false; - } - - CacheSet::iterator cit; - if (isMultiCache()) - { - /* look for subid */ - if ((pit->second).end() == - (cit = (pit->second).find(data.cid.subid))) - { - return false; - } - } - else - { - if ((pit->second).end() == - (cit = (pit->second).find(0))) - { - return false; - } - } - - /* we found it! (cit) */ - data = cit->second; - - return true; -} - - - -bool CacheStore::getAllStoredCaches(std::list &data) -{ - lockData(); /* LOCK MUTEX */ - - std::map::iterator pit; - for(pit = caches.begin(); pit != caches.end(); ++pit) - { - CacheSet::iterator cit; - /* look for subid */ - for(cit = (pit->second).begin(); - cit != (pit->second).end(); ++cit) - { - data.push_back(cit->second); - } - } - - unlockData(); /* UNLOCK MUTEX */ - - return true; -} - - - /* input from CacheStrapper. - * check if we want to download it... - * determine the new name/path - * then request it. - */ -void CacheStore::availableCache(const RsCacheData &data) -{ -#ifdef CS_DEBUG - std::cerr << "CacheStore::availableCache() :" << data << std::endl; -#endif - - /* basic checks */ - lockData(); /* LOCK MUTEX */ - - bool rightCache = (data.cid.type == getCacheType()); - - unlockData(); /* UNLOCK MUTEX */ - - if (!rightCache) - { - return; /* bad id */ - } - - /* These Functions lock the Mutex themselves - */ - - if (!fetchCache(data)) - { - return; /* ignore it */ - } - - RsCacheData rData = data; - - /* get new name */ - if (!nameCache(rData)) - { - return; /* error naming */ - } - - /* request it */ - if(isPeerAcceptedAsCacheProvider(rData.pid)) // Check for service permission - cacheTransfer -> RequestCache(rData, this); - - /* will get callback when it is complete */ - return; -} - - - - /* called when the download is completed ... updates internal data */ -void CacheStore::downloadedCache(const RsCacheData &data) -{ -#ifdef CS_DEBUG - std::cerr << "CacheStore::downloadedCache() :" << data << std::endl; -#endif - - /* updates data */ - if (!loadCache(data)) - { - return; - } -} - /* called when the download is completed ... updates internal data */ -void CacheStore::failedCache(const RsCacheData &data) -{ - (void) data; -#ifdef CS_DEBUG - std::cerr << "CacheStore::failedCache() :" << data << std::endl; -#endif - - return; -} - - - /* virtual function overloaded by cache implementor */ -bool CacheStore::fetchCache(const RsCacheData &data) -{ - /* should we fetch it? */ -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() tofetch?:" << data << std::endl; -#endif - - RsCacheData incache = data; - - lockData(); /* LOCK MUTEX */ - - bool haveCache = ((locked_getStoredCache(incache)) && (data.hash == incache.hash)); - - unlockData(); /* UNLOCK MUTEX */ - - - if (haveCache) - { -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() Already have it: false" << std::endl; -#endif - return false; - } - -#ifdef CS_DEBUG - std::cerr << "CacheStore::fetchCache() Missing this cache: true" << std::endl; -#endif - return true; -} - - -int CacheStore::nameCache(RsCacheData &data) -{ - /* name it... */ - lockData(); /* LOCK MUTEX */ - -#ifdef CS_DEBUG - std::cerr << "CacheStore::nameCache() for:" << data << std::endl; -#endif - - data.name = data.hash.toStdString(); - data.path = getCacheDir(); - -#ifdef CS_DEBUG - std::cerr << "CacheStore::nameCache() done:" << data << std::endl; -#endif - - unlockData(); /* UNLOCK MUTEX */ - - return 1; -} - - -int CacheStore::loadCache(const RsCacheData &data) -{ - /* attempt to load -> dummy function */ -#ifdef CS_DEBUG - std::cerr << "CacheStore::loadCache() Dummy Load for:" << data << std::endl; -#endif - - lockData(); /* LOCK MUTEX */ - - locked_storeCacheEntry(data); - - unlockData(); /* UNLOCK MUTEX */ - - return 1; -} - -/* This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - -void CacheStore::locked_storeCacheEntry(const RsCacheData &data) -{ - /* store what we loaded - overwriting if necessary */ - std::map::iterator pit; - if (caches.end() == (pit = caches.find(data.pid))) - { - /* add in a new CacheSet */ - CacheSet emptySet; - caches[data.pid] = emptySet; - - pit = caches.find(data.pid); - } - - if (isMultiCache()) - { - (pit->second)[data.cid.subid] = data; - } - else - { - (pit->second)[0] = data; - } - - /* tell the strapper we've loaded one */ - if (mStrapper) - { - mStrapper->refreshCacheStore(data); - } - return; -} - - -/********************************* CacheStrapper ********************************* - * This is the bit which handles queries - * - ********************************* CacheStrapper ********************************/ - -CacheStrapper::CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId) - :p3Config(), mServiceCtrl(sc), mFtServiceId(ftServiceId), - csMtx("CacheStrapper") -{ - return; -} - - -void CacheStrapper::addCachePair(CachePair set) -{ - caches[set.id.type] = set; -} - - - /**************** from pqimonclient ********************/ - -void CacheStrapper::statusChange(const std::list &plist) -{ - std::list::const_iterator it; - for(it = plist.begin(); it != plist.end(); ++it) - { - if(it->actions & RS_SERVICE_PEER_CONNECTED) - { - /* grab all the cache ids and add */ - - std::map hashs; - std::map::iterator cit; - - handleCacheQuery(it->id, hashs); - - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - for(cit = hashs.begin(); cit != hashs.end(); ++cit) - { - mCacheUpdates.push_back(std::make_pair(it->id, cit->second)); - } - } - } -} - - /**************** from pqimonclient ********************/ - -void CacheStrapper::refreshCache(const RsCacheData &data,const std::set& destination_peers) -{ - /* we've received an update - * send to all online peers + self intersected with online peers. - */ -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::refreshCache() : " << data << std::endl; -#endif - const RsPeerId& ownid = mServiceCtrl->getOwnId() ; - std::set ids; - - // Need to use ftServiceID as packets are passed through there. - mServiceCtrl->getPeersConnected(mFtServiceId, ids); - ids.insert(ownid) ; - - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - for(std::set::const_iterator it = ids.begin(); it != ids.end(); ++it) - if(destination_peers.find(*it) != destination_peers.end()) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::refreshCache() Send To: " << *it << std::endl; -#endif - mCacheUpdates.push_back(std::make_pair(*it, data)); - } - - IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ -} - -// void CacheStrapper::refreshCache(const RsCacheData &data) -// { -// /* we've received an update -// * send to all online peers + self -// */ -// #ifdef CS_DEBUG -// std::cerr << "CacheStrapper::refreshCache() : " << data << std::endl; -// #endif -// -// std::string ownid = mServiceCtrl->getOwnId() ; -// std::set ids; -// mServiceCtrl->getOnlineList(ids); -// ids.push_back(ownid) ; -// -// { -// RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ -// for(std::set::const_iterator it = ids.begin(); it != ids.end(); ++it) -// if(*it == ownid || isPeerPartipating(*it)) -// mCacheUpdates.push_back(std::make_pair(*it, data)); -// } -// IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ -// } - -void CacheStrapper::refreshCacheStore(const RsCacheData & /* data */ ) -{ - /* indicate to save data */ - IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ - -} - -bool CacheStrapper::getCacheUpdates(std::list > &updates) -{ - RsStackMutex stack(csMtx); /******* LOCK STACK MUTEX *********/ - updates = mCacheUpdates; - mCacheUpdates.clear(); - - return true; -} - - - - /* pass to correct CacheSet */ -void CacheStrapper::recvCacheResponse(RsCacheData &data, time_t /* ts */) -{ - /* find cache store */ - std::map::iterator it2; - if (caches.end() == (it2 = caches.find(data.cid.type))) - { - /* error - don't have this type of cache */ - return; - } - - /* notify the CacheStore */ - (it2 -> second).store -> availableCache(data); - -} - -void CacheStrapper::handleCacheQuery(const RsPeerId& id, std::map &hashs) -{ - /* basic version just iterates through .... - * more complex could decide who gets what! - * - * or that can be handled on a cache by cache basis. - */ - - std::map::iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - (it->second).source -> cachesAvailable(id, hashs); - } - return; -} - -void CacheStrapper::listCaches(std::ostream &out) -{ - /* can overwrite for more control! */ - std::map::iterator it; - out << "CacheStrapper::listCaches() [" << mServiceCtrl->getOwnId(); - out << "] " << " Total Caches: " << caches.size(); - out << std::endl; - for(it = caches.begin(); it != caches.end(); ++it) - { - out << "CacheType: " << it->first; - out << std::endl; - - (it->second).source->listCaches(out); - (it->second).store->listCaches(out); - out << std::endl; - } - return; -} - -void CacheStrapper::listPeerStatus(std::ostream & /* out */) -{ -#if 0 - std::map::iterator it; - out << "CacheStrapper::listPeerStatus() [" << ownId; - out << "] Total Peers: " << status.size() << " Total Caches: " << caches.size(); - out << std::endl; - for(it = status.begin(); it != status.end(); ++it) - { - out << "Peer: " << it->first; - out << " Query: " << (it->second).query; - out << " Answer: " << (it->second).answer; - out << std::endl; - } - return; -#endif -} - - -bool CacheStrapper::findCache(const RsFileHash& hash, RsCacheData &data) const -{ - /* can overwrite for more control! */ - std::map::const_iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - if ((it->second).source->findCache(hash, data)) - { - return true; - } - } - return false; -} - -bool CacheStrapper::CacheExist(RsCacheData& data){ - - std::string filename = data.path + "/" + data.name; - FILE* file = NULL; - file = RsDirUtil::rs_fopen(filename.c_str(), "r"); - - if(file == NULL) - return false; - - fclose(file); - return true; -} - -/***************************************************************************/ -/****************************** CONFIGURATION HANDLING *********************/ -/***************************************************************************/ - -/**** OVERLOADED FROM p3Config ****/ - -RsSerialiser *CacheStrapper::setupSerialiser() -{ - RsSerialiser *rss = new RsSerialiser(); - - /* add in the types we need! */ - rss->addSerialType(new RsCacheConfigSerialiser()); - - return rss; -} - - -bool CacheStrapper::saveList(bool &cleanup, std::list& saveData) -{ - - /* it can delete them! */ - cleanup = true; - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::saveList()" << std::endl; -#endif - - /* iterate through the Caches (local first) */ - - std::list::iterator cit; - std::list ownCaches; - std::list remoteCaches; - const RsPeerId& ownId = mServiceCtrl->getOwnId(); - - std::map::iterator it; - for(it = caches.begin(); it != caches.end(); ++it) - { - std::map::iterator tit; - std::map ownTmp; - (it->second).source -> cachesAvailable(ownId, ownTmp); - (it->second).store -> getAllStoredCaches(remoteCaches); - - for(tit = ownTmp.begin(); tit != ownTmp.end(); ++tit) - { - if(CacheExist(tit->second)) - ownCaches.push_back(tit->second); - } - } - - for(cit = ownCaches.begin(); cit != ownCaches.end(); ++cit) - { - RsCacheConfig *rscc = new RsCacheConfig(); - - // Fixup lazy behaviour in clients... - // This ensures correct loading later. - // (used to be: rscc->pid = cit->pid;) - rscc->pid = ownId; - - //rscc->pname = cit->pname; - rscc->cachetypeid = cit->cid.type; - rscc->cachesubid = cit->cid.subid; - rscc->path = cit->path; - rscc->name = cit->name; - rscc->hash = cit->hash; - rscc->size = cit->size; - rscc->recvd = cit->recvd; - - saveData.push_back(rscc); - } - - for(cit = remoteCaches.begin(); cit != remoteCaches.end(); ++cit) - { - if (cit->pid == ownId) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() discarding Own Remote Cache"; - std::cerr << std::endl; -#endif - continue; /* skip own caches -> will get transferred anyway */ - } - - RsCacheConfig *rscc = new RsCacheConfig(); - - rscc->pid = cit->pid; - //rscc->pname = cit->pname; - rscc->cachetypeid = cit->cid.type; - rscc->cachesubid = cit->cid.subid; - rscc->path = cit->path; - rscc->name = cit->name; - rscc->hash = cit->hash; - rscc->size = cit->size; - rscc->recvd = cit->recvd; - - saveData.push_back(rscc); - } - - /* list completed! */ - return true; -} - - -bool CacheStrapper::loadList(std::list& load) -{ - std::list::iterator it; - RsCacheConfig *rscc; - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Item Count: " << load.size(); - std::cerr << std::endl; -#endif - std::list ownCaches; - std::list remoteCaches; - const RsPeerId& ownId = mServiceCtrl->getOwnId(); - - //peerConnectState ownState; - //mPeerMgr->getOwnNetStatus(ownState); - //std::string ownName = ownState.name+" ("+ownState.location+")"; - - std::map > saveFiles; - std::map >::iterator sit; - - for(it = load.begin(); it != load.end(); ++it) - { - /* switch on type */ - if (NULL != (rscc = dynamic_cast(*it))) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Item: "; - std::cerr << std::endl; - rscc->print(std::cerr, 10); - std::cerr << std::endl; -#endif - RsCacheData cd; - - cd.pid = RsPeerId(rscc->pid) ; - -#if 0 - if(cd.pid == ownId) - { - cd.pname = ownName; - } - else - { - peerConnectState pca; - mPeerMgr->getFriendNetStatus(rscc->pid, pca); - cd.pname = pca.name+" ("+pca.location+")"; - } -#endif - - cd.cid.type = rscc->cachetypeid; - cd.cid.subid = rscc->cachesubid; - cd.path = RsDirUtil::convertPathToUnix(rscc->path); - cd.name = rscc->name; - cd.hash = rscc->hash; - cd.size = rscc->size; - cd.recvd = rscc->recvd; - - /* store files that we want to keep */ - (saveFiles[cd.path]).insert(cd.name); - - std::map::iterator it2; - if (caches.end() == (it2 = caches.find(cd.cid.type))) - { - /* error - don't have this type of cache */ -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Can't Find Cache discarding"; - std::cerr << std::endl; -#endif - } - else - { - /* check that the file exists first.... */ - std::string filename = cd.path; - filename += "/"; - filename += cd.name; -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Checking File: " << filename; - std::cerr << std::endl; -#endif - - uint64_t tmp_size ; - bool fileExists = RsDirUtil::checkFile(filename,tmp_size); - if (fileExists) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Exists ... continuing"; - std::cerr << std::endl; -#endif - if (cd.pid == ownId) - { - - /* load local */ - (it2 -> second).source -> loadLocalCache(cd); -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() loaded Local"; - std::cerr << std::endl; -#endif - } - else - { - /* load remote */ - (it2 -> second).store -> loadCache(cd); -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() loaded Remote"; - std::cerr << std::endl; -#endif - } - } - else - { - std::cerr << "CacheStrapper::loadList() Doesn't Exist dropping: " << filename; - std::cerr << std::endl; - } - - } - - /* cleanup */ - delete (*it); - - } - else - { - /* cleanup */ - delete (*it); - } - } - load.clear() ; - - /* assemble a list of dirs to clean (union of cache dirs) */ - std::list cacheDirs; - std::list::iterator dit; -#ifdef CS_DEBUG - std::set::iterator fit; -#endif - std::map::iterator cit; - for(cit = caches.begin(); cit != caches.end(); ++cit) - { - std::string lcdir = (cit->second).source->getCacheDir(); - std::string rcdir = (cit->second).store->getCacheDir(); - - if (cacheDirs.end() == std::find(cacheDirs.begin(), cacheDirs.end(), lcdir)) - { - cacheDirs.push_back(lcdir); - } - - if (cacheDirs.end() == std::find(cacheDirs.begin(), cacheDirs.end(), rcdir)) - { - cacheDirs.push_back(rcdir); - } - } - -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Files To Save:" << std::endl; -#endif - -#ifdef CS_DEBUG - for(sit = saveFiles.begin(); sit != saveFiles.end(); ++sit) - { - std::cerr << "CacheStrapper::loadList() Files To Save in dir: <" << sit->first << ">" << std::endl; - for(fit = (sit->second).begin(); fit != (sit->second).end(); ++fit) - { - std::cerr << "\tFile: " << *fit << std::endl; - } - } -#endif - - std::set emptySet; - for(dit = cacheDirs.begin(); dit != cacheDirs.end(); ++dit) - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() Cleaning cache dir: <" << *dit << ">" << std::endl; -#endif - sit = saveFiles.find(RsDirUtil::convertPathToUnix(*dit)); - - if (sit != saveFiles.end()) - { -#ifdef CS_DEBUG - for(fit = (sit->second).begin(); fit != (sit->second).end(); ++fit) - { - std::cerr << "CacheStrapper::loadList() Keeping File: " << *fit << std::endl; - } -#endif - RsDirUtil::cleanupDirectoryFaster(*dit, sit->second); - } - else - { -#ifdef CS_DEBUG - std::cerr << "CacheStrapper::loadList() No Files to save here!" << std::endl; -#endif - RsDirUtil::cleanupDirectoryFaster(*dit, emptySet); - } - } - - return true; - -} - - -/********************************* CacheStrapper ********************************* - * This is the bit which handles queries - * - ********************************* CacheStrapper ********************************/ - - -/* request from CacheStore */ -bool CacheTransfer::RequestCache(RsCacheData &data, CacheStore *cbStore) -{ - /* check for a previous request -> and cancel - * - * - if duplicate pid, cid -> cancel old transfer - * - if duplicate hash -> Fail Transfer - */ - - std::map::iterator dit; - std::map::iterator sit; - - for(dit = cbData.begin(); dit != cbData.end(); ++dit) - { - if (((dit->second).pid == data.pid) && - ((dit->second).cid.type == data.cid.type) && - ((dit->second).cid.subid == data.cid.subid)) - { - sit = cbStores.find(dit->second.hash); - - /* if identical to previous request, then we don't want to cancel - * a partially transferred cache file - * - * We wouldn't expect to have to request it again, however the feedback loop - * from ftController is not completed (it should callback and tell us if it cancels - * the cache file. XXX TO FIX. - */ - if ((data.hash == dit->second.hash) && - (data.path == dit->second.path) && - (data.size == dit->second.size)) - { - std::cerr << "Re-request duplicate cache... let it continue"; - std::cerr << std::endl; - /* request data */ - RequestCacheFile(data.pid, data.path, data.hash, data.size); - - return true; - } - - /* cancel old transfer */ - CancelCacheFile(dit->second.pid, dit->second.path, - dit->second.hash, dit->second.size); - - sit = cbStores.find(dit->second.hash); - cbData.erase(dit); - cbStores.erase(sit); - - break; - } - } - - /* find in store.... */ - sit = cbStores.find(data.hash); - if (sit != cbStores.end()) - { - /* Duplicate Current Request */ - cbStore -> failedCache(data); - return false; - } - - - /* store request */ - cbData[data.hash] = data; - cbStores[data.hash] = cbStore; - - /* request data */ - RequestCacheFile(data.pid, data.path, data.hash, data.size); - - /* wait for answer */ - return true; -} - - -/* to be overloaded */ -bool CacheTransfer::RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size) -{ - (void) id; - (void) path; - (void) size; -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::RequestCacheFile() : from:" << id << " #"; - std::cerr << hash << " size: " << size; - std::cerr << " savepath: " << path << std::endl; - std::cerr << "CacheTransfer::RequestCacheFile() Dummy... saying completed"; - std::cerr << std::endl; -#endif - - /* just tell them we've completed! */ - CompletedCache(hash); - return true; -} - -/* to be overloaded */ -bool CacheTransfer::CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash &hash, uint64_t size) -{ - (void) id; - (void) path; - (void) hash; - (void) size; -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CancelCacheFile() : from:" << id << " #"; - std::cerr << hash << " size: " << size; - std::cerr << " savepath: " << path << std::endl; - std::cerr << "CacheTransfer::CancelCacheFile() Dummy fn"; - std::cerr << std::endl; -#endif - - return true; -} - - -/* internal completion -> does cb */ -bool CacheTransfer::CompletedCache(const RsFileHash& hash) -{ - std::map::iterator dit; - std::map::iterator sit; - -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache(" << hash << ")"; - std::cerr << std::endl; -#endif - - /* find in store.... */ - sit = cbStores.find(hash); - dit = cbData.find(hash); - - if ((sit == cbStores.end()) || (dit == cbData.end())) - { -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache() Failed to find it"; - std::cerr << std::endl; -#endif - - return false; - } - -#ifdef CS_DEBUG - std::cerr << "CacheTransfer::CompletedCache() callback to store"; - std::cerr << std::endl; -#endif - /* callback */ - (sit -> second) -> downloadedCache(dit->second); - - /* clean up store */ - cbStores.erase(sit); - cbData.erase(dit); - - return true; -} - -/* internal completion -> does cb */ -bool CacheTransfer::FailedCache(const RsFileHash& hash) -{ - std::map::iterator dit; - std::map::iterator sit; - - /* find in store.... */ - sit = cbStores.find(hash); - dit = cbData.find(hash); - - if ((sit == cbStores.end()) || (dit == cbData.end())) - { - return false; - } - - /* callback */ - (sit -> second) -> failedCache(dit->second); - - /* clean up store */ - cbStores.erase(sit); - cbData.erase(dit); - - return true; -} - - -bool CacheTransfer::FindCacheFile(const RsFileHash &hash, std::string &path, uint64_t &size) -{ - RsCacheData data; - if (strapper->findCache(hash, data)) - { - path = data.path + "/" + data.name; - size = data.size; - return true; - } - - return false; -} - - - diff --git a/libretroshare/src/dbase/cachestrapper.h b/libretroshare/src/dbase/cachestrapper.h deleted file mode 100644 index 3be595fc4..000000000 --- a/libretroshare/src/dbase/cachestrapper.h +++ /dev/null @@ -1,475 +0,0 @@ -/* - * RetroShare FileCache Module: cachestrapper.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_CACHE_STRAPPER_H -#define MRK_CACHE_STRAPPER_H - -#include "pqi/p3cfgmgr.h" -#include "pqi/pqiservicemonitor.h" -#include "util/rsthreads.h" - -#include -#include -#include -#include -#include - -/******************* CacheStrapper and Related Classes ******************* - * A generic Cache Update system. - * - * CacheStrapper: maintains a set of CacheSources, and CacheStores, - * queries and updates as new information arrives. - * - * CacheTransfer: Interface for FileTransfer Class to support. - * - * CacheSource: Base Class for cache data provider. eg. FileIndexMonitor. - * CacheStore: Base Class for data cache. eg. FileCache/Store. - * - * Still TODO: - * (1) Design and Implement the Upload side of CacheTransfer/CacheStrapper. - * (2) CacheStrapper:: Save / Load Cache lists.... - * (3) Clean up lists, maps on shutdown etc. - * (4) Consider Mutexes for multithreaded operations. - * (5) Test the MultiSource/Store capabilities. - * - ******************* CacheStrapper and Related Classes *******************/ - - -class CacheTransfer; /* Interface for File Transfer */ -class CacheSource; /* Interface for local File Index/Monitor */ -class CacheStore; /* Interface for the actual Cache */ -class CacheStrapper; /* Controlling Class */ - -/******************************** CacheId ********************************/ - -/*! - * Use this to identify the type of cache source, strapper, - */ -class CacheId -{ - public: - CacheId() :type(0), subid(0) { return; } - CacheId(uint16_t a, uint16_t b) :type(a), subid(b) { return; } - uint16_t type; /// cache types, this should be set to services type - uint16_t subid; /// should be initialised when using multicache feature of cachestrapper -}; - - -bool operator<(const CacheId &a, const CacheId &b); - -/*! - * Use for identifying physical files that have been chosen as cache data - * note: this does not actual store the data but serves to locate on network (via hash attribute, - * and on file via path) - */ -class RsCacheData -{ - public: - - RsPeerId pid; /// peer id - /// REMOVED as a WASTE to look it up everywhere! std::string pname; /// peer name (can be used by cachestore) - CacheId cid; /// cache id - std::string path; /// file system path where physical cache data is located - std::string name; - RsFileHash hash; - uint64_t size; - time_t recvd; /// received timestamp -}; - - -std::ostream &operator<<(std::ostream &out, const RsCacheData &d); - -/***************************** CacheTransfer *****************************/ - -/** - * Interface for FileTransfer Class to support cache - */ -class CacheTransfer -{ - public: - - CacheTransfer(CacheStrapper *cs) :strapper(cs) { return; } - virtual ~CacheTransfer() {} - - /*! - * upload side of things .... searches through CacheStrapper. - */ - bool FindCacheFile(const RsFileHash& hash, std::string &path, uint64_t &size); - - /*! - * At the download side RequestCache() => overloaded RequestCacheFile() - * the class should then call CompletedCache() or FailedCache() - */ - bool RequestCache(RsCacheData &data, CacheStore *cbStore); /* request from CacheStore */ - - protected: - - /*! - * to be overloaded - */ - virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size); - - bool CompletedCache(const RsFileHash &hash); /* internal completion -> does cb */ - bool FailedCache(const RsFileHash &hash); /* internal completion -> does cb */ - - private: - - CacheStrapper *strapper; - - std::map cbData; - std::map cbStores; -}; - - - -/************************ CacheSource/CacheStore *************************/ - -typedef std::map CacheSet; - -/*! - * Implements features needed for a service to act as a cachesource and allow pushing a of cache data from service to strapper - * Service is able to use this class for refresh its cache (push cache data) - * and interface to load and check cache availablility among peers (source of cache data) - * Architecturally Cachestrapper maintains the cachesource (which is passed as a pointer handle) while the cachesource-inheriting - * service can update cachesource as to new cache sources (cache data) created. Equivalently it enquiries through cache source for - * new cache data from peers - * @see p3Distrib - */ -class CacheSource -{ - public: - - CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir); - virtual ~CacheSource() {} - - /*! - * called to determine available cache for peer - - * default acceptable (returns all) - */ - virtual bool cachesAvailable(const RsPeerId& pid, std::map &ids)=0; - - /*! - * function called at startup to load from - * configuration file.... - * to be overloaded by inherited class - */ - virtual bool loadLocalCache(const RsCacheData &data); - - /* control Caches available */ - bool refreshCache(const RsCacheData &data,const std::set& destination_peers); - bool refreshCache(const RsCacheData &data); - bool clearCache(CacheId id); - - /* controls if peer is an accepted receiver for cache items. Default is yes. To be overloaded. */ - virtual bool isPeerAcceptedAsCacheReceiver(const RsPeerId& /*peer_id*/) { return true ; } - - /* get private data */ - std::string getCacheDir() { return cacheDir; } - bool isMultiCache() { return multiCache; } - uint16_t getCacheType() { return cacheType; } - - /* display */ - void listCaches(std::ostream &out); - - /* search */ - bool findCache(const RsFileHash& hash, RsCacheData &data) const; - - protected: - - uint16_t cacheType; /// for checking of cache type (usually of child class of source) - bool multiCache; /// whether multisource is in use or not. - CacheStrapper *mStrapper; - - /*** MUTEX LOCKING */ - void lockData() const; - void unlockData() const; - - CacheSet caches; /// all local cache data stored here - std::map mOldCaches; /// replaced/cleared caches are pushed here (in case requested) - - private: - - std::string cacheDir; - mutable RsMutex cMutex; - -}; - - -/*! - * Base Class for data cache. eg. FileCache/Store. - * This is best used to deal with external caches from other peers - * @see p3Distrib. pqiMonitor - */ -class CacheStore -{ - public: - - /*! - * - * @param t set to particular rs_service id. see rsserviceids.h - * @param m whether this is multicache service (true) or not(false) - * @param cs cache strapper instance responsible for maintaining the cache service - * @param cft cache transfer instance responsible for rquestiing and tranfering caches - * @param cachedir directory used to store cache related info for cachestore client - * @return - */ - CacheStore(uint16_t t, bool m, CacheStrapper *cs, CacheTransfer *cft, std::string cachedir); - virtual ~CacheStore() {} - - /* current stored data */ - - /*! - * - * @param data returns cache data for pid/cid set in data itself - * @return false is unsuccessful and vice versa - */ - bool getStoredCache(RsCacheData &data); /* use pid/cid in data */ - - /*! - * - * @param data all cache store by cachestore is store here - * @return false not returned, only true at the moment - */ - bool getAllStoredCaches(std::list &data); /* use pid/cid in data */ - - /*! - * input from CacheStrapper -> store can then download new data - */ - void availableCache(const RsCacheData &data); - - /*! - * should be called when the download is completed ... cache data is loaded - */ - void downloadedCache(const RsCacheData &data); - - /*! - * called if the download fails, TODO: nothing done yet - */ - void failedCache(const RsCacheData &data); - - /* virtual functions overloaded by cache implementor */ - - /* controls if peer is an accepted provider for cache items. Default is yes. To be overloaded. */ - virtual bool isPeerAcceptedAsCacheProvider(const RsPeerId& /*peer_id*/) { return true ; } - - /*! - * @param data cache data is stored here - * @return false is failed (cache does not exist), otherwise true - */ - virtual bool fetchCache(const RsCacheData &data); /* a question? */ - virtual int nameCache(RsCacheData &data); /* fill in the name/path */ - virtual int loadCache(const RsCacheData &data); /* actual load, once data available */ - - /* get private data */ - - std::string getCacheDir() { return cacheDir; } - bool isMultiCache() { return multiCache; } - uint16_t getCacheType() { return cacheType; } - - /*! - * display, e.g. can pass std::out, cerr, ofstream, etc - */ - void listCaches(std::ostream &out); - - protected: - - /*! - * ** MUTEX LOCKING - */ - void lockData() const; - - /*! - * ** MUTEX LOCKING - */ - void unlockData() const; - - /*! This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - void locked_storeCacheEntry(const RsCacheData &data); - - /*! This function is called to store Cache Entry in the CacheStore Table. - * it must be called from within a Mutex Lock.... - * - * It doesn't lock itself -> to avoid race conditions - */ - bool locked_getStoredCache(RsCacheData &data); - - private: - - uint16_t cacheType; /* for checking */ - bool multiCache; /* do we care about subid's */ - - CacheStrapper *mStrapper; - CacheTransfer *cacheTransfer; - - std::string cacheDir; - - mutable RsMutex cMutex; - std::map caches; - -}; - - - -/***************************** CacheStrapper *****************************/ - - -/*! - * a convenient to pass cache handles to cachestrapper to maintain the cache service - * Make Sure you get the Ids right! see rsservicesids.h. - * When creating a cache service this data structure is - * source, usually the child class of store and source also serves as both handles - * @see CacheStrapper - */ -class CachePair -{ - public: - - /*! - * Default constructor, all variables set to NULL - */ - CachePair() - :source(NULL), store(NULL), id(0, 0) { return; } - - /*! - * - * @param a cache source for service - * @param b cache store for service - * @param c the cache service id, c.type should be set to service id service-child class of store and source - */ - CachePair(CacheSource *a, CacheStore *b, CacheId c) - :source(a), store(b), id(c) { return; } - - CacheSource *source; - CacheStore *store; - CacheId id; /// should be set id type to service types of store and source service-child class, and subid for multicache use -}; - - -bool operator<(const CachePair &a, const CachePair &b); - -/*! - * CacheStrapper: maintains a set of CacheSources, and CacheStores, - * queries and updates as new information arrives. - */ - -class p3ServiceControl; - -class CacheStrapper: public pqiServiceMonitor, public p3Config -{ - public: - - /*! - * @param cm handle used by strapper for getting peer connection information (online peers, sslids...) - * @return - */ - CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId); -virtual ~CacheStrapper() { return; } - - /************* from pqiMonitor *******************/ -virtual void statusChange(const std::list &plist); - /************* from pqiMonitor *******************/ - - /* Feedback from CacheSources */ - -/*! - * send data to peers online and self - * @param data - * - */ -void refreshCache(const RsCacheData &data); -void refreshCache(const RsCacheData &data,const std::set& destination_peers); // specify a particular list of destination peers (self not added!) - -/*! - * forces config savelist - * @param data - * @see saveList() - */ -void refreshCacheStore(const RsCacheData &data); - -/*! - * list of Caches to send out - */ -bool getCacheUpdates(std::list > &updates); - -/*! - * add to strapper's cachepair set so a related service's store and source can be maintained - * @param pair the source and store handle for a service - */ -void addCachePair(CachePair pair); - - /*** I/O (2) ***/ -void recvCacheResponse(RsCacheData &data, time_t ts); -void handleCacheQuery(const RsPeerId& id, std::map &data); - - -/*! - * search through CacheSources. - * @return false if cachedate mapping to hash not found - */ -bool findCache(const RsFileHash &hash, RsCacheData &data) const; - - /* display */ -void listCaches(std::ostream &out); - -/*! - * does not do anything - * @param out - * @deprecated - */ -void listPeerStatus(std::ostream &out); - -/** - * Checks if the cache physically exist at path given - * @param data - * @return whether it exists or not - */ -bool CacheExist(RsCacheData& data); - - /* Config */ - protected: - - /* Key Functions to be overloaded for Full Configuration */ -virtual RsSerialiser *setupSerialiser(); -virtual bool saveList(bool &cleanup, std::list&); -virtual bool loadList(std::list& load); - - private: - - /* these are static - so shouldn't need mutex */ - p3ServiceControl *mServiceCtrl; - uint32_t mFtServiceId; - - std::map caches; - - RsMutex csMtx; /* protect below */ - - std::list > mCacheUpdates; -}; - - -#endif diff --git a/libretroshare/src/dbase/cachetest.h b/libretroshare/src/dbase/cachetest.h deleted file mode 100644 index 59d55b5be..000000000 --- a/libretroshare/src/dbase/cachetest.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * RetroShare FileCache Module: cachetest.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_TEST_CACHE_H -#define MRK_TEST_CACHE_H - -#include "cachestrapper.h" - -#define TESTID 0xffff -#define TESTID2 0xffee - -class CacheTestSource: public CacheSource -{ - public: - CacheTestSource(CacheStrapper *cs, std::string dir) - :CacheSource(cs, TESTID, false, dir) { return; } -}; - -class CacheTestStore: public CacheStore -{ - public: - CacheTestStore(CacheTransfer *cft, std::string dir) - :CacheStore(TESTID, false, cft, dir) { return; } -}; - - -class CacheTestMultiSource: public CacheSource -{ - public: - CacheTestMultiSource(CacheStrapper *cs, std::string dir) - :CacheSource(cs, TESTID2, true, dir) { return; } -}; - -class CacheTestMultiStore: public CacheStore -{ - public: - CacheTestMultiStore(CacheTransfer *cft, std::string dir) - :CacheStore(TESTID2, true, cft, dir) { return; } -}; - - -#endif - diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc deleted file mode 100644 index 3269469f7..000000000 --- a/libretroshare/src/dbase/fimonitor.cc +++ /dev/null @@ -1,1706 +0,0 @@ -/* - * RetroShare FileCache Module: fimonitor.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifdef WINDOWS_SYS -#include "util/rsstring.h" -#include "util/rswin.h" -#endif - -#include "rsserver/p3face.h" -#include "dbase/fimonitor.h" -#include "util/rsdir.h" -#include "util/rsmemory.h" -#include "pqi/authssl.h" -#include "serialiser/rsserviceids.h" -#include "retroshare/rsiface.h" -#include "pqi/p3notify.h" -#include "retroshare/rspeers.h" -#include "retroshare/rstypes.h" -#include "util/folderiterator.h" -#include - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// *********** -//#define FIM_DEBUG 1 -// ***********/ - -FileIndexMonitor::FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid,const std::string& config_dir) - :CacheSource(RS_SERVICE_TYPE_FILE_INDEX, true, cs, cachedir), fiMutex("FileIndexMonitor"), fi(pid), - pendingDirs(false), pendingForceCacheWrite(false), - mForceCheck(false), mInCheck(false), hashCache(config_dir+"/" + "file_cache"),useHashCache(true) - -{ - updatePeriod = 15 * 60; // 15 minutes - reference_time = 0 ; -} - -bool FileIndexMonitor::autoCheckEnabled() const -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return updatePeriod > 0 ; -} - -bool FileIndexMonitor::rememberHashFiles() -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return useHashCache ; -} -void FileIndexMonitor::setRememberHashFiles(bool b) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ -#ifdef FIM_DEBUG - std::cerr << "Setting useHashCache to " << b << std::endl; -#endif - useHashCache = b ; -} -void FileIndexMonitor::setRememberHashFilesDuration(uint32_t days) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - -#ifdef FIM_DEBUG - std::cerr << "Setting HashCache duration to " << days << std::endl; -#endif - hashCache.setRememberHashFilesDuration(days) ; -} - -uint32_t FileIndexMonitor::rememberHashFilesDuration() const -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - return hashCache.rememberHashFilesDuration() ; -} - -// Remove any memory of formerly hashed files that are not shared anymore -void FileIndexMonitor::clearHashFiles() -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - hashCache.clear() ; - hashCache.save() ; -} - -HashCache::HashCache(const std::string& path) - : _path(path) -{ - _max_cache_duration_days = 10 ; // 10 days is the default value. - _files.clear() ; - _changed = false ; - - // check for unencrypted - - std::istream *f = NULL ; - uint64_t file_size ; - - if(RsDirUtil::checkFile( _path+".bin",file_size,false ) ) - { - std::cerr << "Encrypted hash cache file present. Reading it." << std::endl; - - // read the binary stream into memory. - // - void *buffer = rs_malloc(file_size) ; - - if(buffer == NULL) - return ; - - FILE *F = fopen( (_path+".bin").c_str(),"rb") ; - if (!F) - { - std::cerr << "Cannot open file for reading encrypted file cache, filename " << (_path+".bin") << std::endl; - free(buffer); - return; - } - if(fread(buffer,1,file_size,F) != file_size) - { - std::cerr << "Cannot read from file " + _path+".bin" << ": something's wrong." << std::endl; - free(buffer) ; - fclose(F) ; - return ; - } - fclose(F) ; - - void *decrypted_data =NULL; - int decrypted_data_size =0; - - if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) - { - std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; - free(buffer) ; - return ; - } - free(buffer) ; - - std::string s((char *)decrypted_data,decrypted_data_size) ; - f = new std::istringstream(s) ; - - free(decrypted_data) ; - } - else - { - std::cerr << "Encrypted file cache not present. Trying unencrypted..." << std::endl; - - f = new std::ifstream( (_path+".lst").c_str()) ; - - if(!f->good()) - { - delete f ; - std::cerr << "Unencrypted file cache not present either." << std::endl; - return ; - } - } - - std::streamsize max_line_size = 2000 ; // should be enough. Anyway, if we - // miss one entry, we just lose some - // cache itemsn but this is not too - // much of a problem. - char *buff = new char[max_line_size] ; - -#ifdef FIM_DEBUG - std::cerr << "Loading HashCache from file " << path << std::endl ; - int n=0 ; -#endif - - while(!f->eof()) - { - HashCacheInfo info ; - - f->getline(buff,max_line_size,'\n') ; - std::string name(buff) ; - - f->getline(buff,max_line_size,'\n') ; //if(sscanf(buff,"%llu",&info.size) != 1) break ; - - info.size = 0 ; - sscanf(buff, RsDirUtil::scanf_string_for_uint(sizeof(info.size)), &info.size); - - f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.time_stamp)),&info.time_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } - f->getline(buff,max_line_size,'\n') ; if(sscanf(buff,RsDirUtil::scanf_string_for_uint(sizeof(info.modf_stamp)),&info.modf_stamp) != 1) { std::cerr << "Could not read one entry! Giving up." << std::endl; break ; } - f->getline(buff,max_line_size,'\n') ; info.hash = RsFileHash(std::string(buff)) ; - -#ifdef FIM_DEBUG - std::cerr << " (" << name << ", " << info.size << ", " << info.time_stamp << ", " << info.modf_stamp << ", " << info.hash << std::endl ; - ++n ; -#endif - _files[name] = info ; - } - - delete[] buff ; - delete f ; -#ifdef FIM_DEBUG - std::cerr << n << " entries loaded." << std::endl ; -#endif -} - -void HashCache::save() -{ - if(!_changed) - { -#ifdef FIM_DEBUG - std::cerr << "Hash cache not changed. Not saving." << std::endl ; -#endif - return; - } - -#ifdef FIM_DEBUG - std::cerr << "Saving Hash Cache to file " << _path << "..." << std::endl ; -#endif - - std::ostringstream f ; - for(std::map::const_iterator it(_files.begin()); - it != _files.end(); - ++it) - { - f << it->first << std::endl ; - f << it->second.size << std::endl; - f << it->second.time_stamp << std::endl; - f << it->second.modf_stamp << std::endl; - f << it->second.hash << std::endl; - } - - void *encryptedData = NULL ; - int encDataLen = 0 ; - - if(!AuthSSL::getAuthSSL()->encrypt( - encryptedData, - encDataLen, - f.str().c_str(), - f.str().length(), - AuthSSL::getAuthSSL()->OwnId())) - { - std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; - return; - } - - FILE *F = fopen( (_path+".bin.tmp").c_str(),"wb" ) ; - - if(!F) - { - std::cerr << "Cannot open encrypted file cache for writing: " << _path+".bin.tmp" << std::endl; - goto save_free; - } - if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) - { - std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; - fclose(F) ; - goto save_free; - } - - fclose(F) ; - - RsDirUtil::renameFile(_path+".bin.tmp",_path+".bin") ; -#ifdef FIM_DEBUG - std::cerr << "done." << std::endl ; -#endif - - _changed = false; - -save_free: - free(encryptedData); -} - -bool HashCache::find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) -{ -#ifdef FIM_DEBUG - std::cerr << "HashCache: looking for " << full_path << std::endl ; -#endif - time_t now = time(NULL) ; - std::map::iterator it(_files.find(full_path)) ; - - if(it != _files.end() && (uint64_t)time_stamp == it->second.modf_stamp && size == it->second.size) - { - hash = it->second.hash ; - it->second.time_stamp = now ; -#ifdef FIM_DEBUG - std::cerr << "Found in cache." << std::endl ; -#endif - return true ; - } - else - { -#ifdef FIM_DEBUG - std::cerr << "not found in cache." << std::endl ; -#endif - return false ; - } -} -void HashCache::insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) -{ - HashCacheInfo info ; - info.size = size ; - info.modf_stamp = time_stamp ; - info.time_stamp = time(NULL) ; - info.hash = hash ; - - _files[full_path] = info ; - _changed = true ; - -#ifdef FIM_DEBUG - std::cerr << "Entry inserted in cache: " << full_path << ", " << size << ", " << time_stamp << std::endl ; -#endif -} -void HashCache::clean() -{ -#ifdef FIM_DEBUG - std::cerr << "Cleaning HashCache..." << std::endl ; -#endif - time_t now = time(NULL) ; - time_t duration = _max_cache_duration_days * 24 * 3600 ; // seconds - -#ifdef FIM_DEBUG - std::cerr << "cleaning hash cache." << std::endl ; -#endif - - for(std::map::iterator it(_files.begin());it!=_files.end();) - if(it->second.time_stamp + duration < (uint64_t)now) - { -#ifdef FIM_DEBUG - std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ; -#endif - std::map::iterator tmp(it) ; - ++tmp ; - _files.erase(it) ; - it=tmp ; - _changed = true ; - } - else - ++it ; - -#ifdef FIM_DEBUG - std::cerr << "Done." << std::endl; -#endif -} - -FileIndexMonitor::~FileIndexMonitor() -{ - /* Data cleanup - TODO */ -} - -int FileIndexMonitor::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) -{ - results.clear(); - std::list firesults; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - fi.searchTerms(keywords, firesults); - } - - return filterResults(firesults,results,flags,peer_id) ; -} - -int FileIndexMonitor::SearchBoolExp(Expression *exp, std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const -{ - results.clear(); - std::list firesults; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - fi.searchBoolExp(exp, firesults); - } - - return filterResults(firesults,results,flags,peer_id) ; -} - -int FileIndexMonitor::filterResults(std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const -{ -#ifdef DEBUG - if((flags & ~RS_FILE_HINTS_PERMISSION_MASK) > 0) - std::cerr << "(EE) ***** FileIndexMonitor:: Flags ERROR in filterResults!!" << std::endl; -#endif - /* translate/filter results */ - - for(std::list::const_iterator rit(firesults.begin()); rit != firesults.end(); ++rit) - { - DirDetails cdetails ; - RequestDirDetails (*rit,cdetails,FileSearchFlags(0u)); -#ifdef FIM_DEBUG - std::cerr << "Filtering candidate " << (*rit)->name << ", flags=" << cdetails.flags << ", peer=" << peer_id ; -#endif - - if(!peer_id.isNull()) - { - FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(peer_id,cdetails.flags,cdetails.parent_groups) ; - - if (cdetails.type == DIR_TYPE_FILE && ( permission_flags & flags )) - { - cdetails.id.clear() ; - results.push_back(cdetails); -#ifdef FIM_DEBUG - std::cerr << ": kept" << std::endl ; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << ": discarded" << std::endl ; -#endif - } - else - results.push_back(cdetails); - } - - return !results.empty() ; -} - -bool FileIndexMonitor::findLocalFile(const RsFileHash& hash,FileSearchFlags hint_flags, const RsPeerId& peer_id,std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const -{ - std::list results; - bool ok = false; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - -#ifdef FIM_DEBUG -// std::cerr << "FileIndexMonitor::findLocalFile() Hash: " << hash << std::endl; -#endif - /* search through the fileIndex */ - fi.searchHash(hash, results); - - if ( !results.empty() ) - { - /* find the full path for the first entry */ - FileEntry *fe = results.front(); - DirEntry *de = fe->parent; /* all files must have a valid parent! */ - - locked_findShareFlagsAndParentGroups(fe,storage_flags,parent_groups) ; - - // turn share flags into hint flags - - FileSearchFlags shflh = peer_id.isNull()?(RS_FILE_HINTS_BROWSABLE|RS_FILE_HINTS_NETWORK_WIDE):rsPeers->computePeerPermissionFlags(peer_id,storage_flags,parent_groups) ; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile: Filtering candidate " << fe->name << ", flags=" << storage_flags << ", hint_flags=" << hint_flags << ", peer_id = " << peer_id << std::endl ; -#endif - - if(peer_id.isNull() || (shflh & hint_flags)) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile() Found Name: " << fe->name << std::endl; -#endif - std::string shpath = RsDirUtil::removeRootDir(de->path); - std::string basedir = RsDirUtil::getRootDir(de->path); - std::string realroot = locked_findRealRoot(basedir); - - /* construct full name */ - if (realroot.length() > 0) - { - fullpath = realroot + "/"; - if (shpath != "") - { - fullpath += shpath + "/"; - } - fullpath += fe->name; - - size = fe->size; - ok = true; - } -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::findLocalFile() Found Path: " << fullpath << std::endl; - std::cerr << "FileIndexMonitor::findLocalFile() Found Size: " << size << std::endl; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << "FileIndexMonitor::findLocalFile() discarded" << std::endl ; -#endif - } - } /* UNLOCKED DIRS */ - - return ok; -} - -bool FileIndexMonitor::convertSharedFilePath(std::string path, std::string &fullpath) -{ - bool ok = false; - - { - RsStackMutex stackM(fiMutex) ;/* LOCKED DIRS */ - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::convertSharedFilePath() path: " << path << std::endl; -#endif - - std::string shpath = RsDirUtil::removeRootDir(path); - std::string basedir = RsDirUtil::getRootDir(path); - std::string realroot = locked_findRealRoot(basedir); - - /* construct full name */ - if (realroot.length() > 0) - { - fullpath = realroot + "/"; - fullpath += shpath; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::convertSharedFilePath() Found Path: " << fullpath << std::endl; -#endif - ok = true; - } - - } /* UNLOCKED DIRS */ - - return ok; -} - - -bool FileIndexMonitor::loadLocalCache(const RsCacheData &data) /* called with stored data */ -{ - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadLocalCache(): subid = " << data.cid.subid << ", filename=" << data.name << ", peer id = " << data.pid << std::endl; -#endif - - if(!strcmp(data.name.c_str()+data.name.size()-5,".rsfc"))// this trick allows to load the complete file. Not the one being shared. - { // other files are discarded and re-created in case permissions have changed. - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - - /* More error checking needed here! */ - - std::string name = data.name ; - - if ( fi.loadIndex(data.path + '/' + name, RsFileHash(), data.size) ) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadCache() Success!"; - std::cerr << std::endl; -#endif - fi.root->row = 0; - fi.root->name = data.pid.toStdString(); // XXX Hack here - TODO - - std::string fname_browsable = data.path + '/' + name ; - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(fname_browsable, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(fname_browsable.c_str(), &buf)) -#endif - { - reference_time = buf.st_mtime ; -#ifdef FIM_DEBUG - std::cerr << "Read new reference time of created file " << fname_browsable << ", to " << reference_time << std::endl; -#endif - } - else - { - std::cerr << "(EE) Error. Cannot get the proper modification time for file " << fname_browsable << " errno=" << errno << std::endl; - reference_time = 0 ; - } -#ifdef FIM_DEBUG - std::cerr << "Current reference time is now : " << reference_time << std::endl; -#endif - } - else - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::loadCache() Failed!"; - std::cerr << std::endl; -#endif - reference_time = 0 ; - } - - fi.updateMaxModTime() ; - - // The index is re-saved. - // - we might have new friends - // - the cache system removes old cache items so we need to re-create it. - // - locked_saveFileIndexes(false) ; - } -#ifdef FIM_DEBUG - else - std::cerr << "FileIndexMonitor:: not loading cache item " << data.name << std::endl; -#endif - - return false; -} - -bool FileIndexMonitor::updateCache(const RsCacheData &data,const std::set& destination_peers) /* we call this one */ -{ - return refreshCache(data,destination_peers); -} - - -int FileIndexMonitor::getPeriod() const -{ -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setPeriod() getting watch period" << std::endl; -#endif - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - return updatePeriod ; -} - -void FileIndexMonitor::setPeriod(int period) -{ - RsStackMutex mtx(fiMutex) ; /* LOCKED DIRS */ - updatePeriod = period; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setPeriod() Setting watch period to " << updatePeriod << std::endl; -#endif -} - -void FileIndexMonitor::data_tick() -{ - if(autoCheckEnabled()) - updateCycle(); - - int i=0 ; - for(;;++i) - { - if(!isRunning()) - return; - - usleep(1*1000*1000); // 1 sec - - /* check dirs if they've changed */ - if (internal_setSharedDirectories()) - break; - - { - RsStackMutex mtx(fiMutex) ; - - if(i >= abs(updatePeriod)) - break ; - } - } - - if(i < abs(updatePeriod) || autoCheckEnabled()) - updateCycle(); -} - -void FileIndexMonitor::updateCycle() -{ - time_t startstamp = time(NULL); - -#ifdef FIM_DEBUG - std::cerr << "Checking directory for new/modified files. Reference time is " << reference_time << std::endl; -#endif - /* iterate through all out-of-date directories */ - bool moretodo = true; - bool fiMods = false; - - { - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - mInCheck = true; - } - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_EXAMINING_FILES, "") ; - - std::vector to_hash ; - - bool cache_is_new ; - { - RsStackMutex mtx(fiMutex) ; - cache_is_new = useHashCache && hashCache.empty() ; - } - struct stat64 buf; - - while(isRunning() && moretodo) - { - /* sleep a bit for each loop */ -// csoler: I'm disabling this since it causes a very long update cycle when the number -// of directories to go through is very large. -// -// usleep(100*1000); /* 100 msec */ - - /* check if directories have been updated */ - - if (internal_setSharedDirectories()) /* reset start time */ - startstamp = time(NULL); - - /* Handle a Single out-of-date directory */ - - time_t stamp = time(NULL); - - /* lock dirs from now on */ - RsStackMutex mtx(fiMutex) ; - - DirEntry *olddir = fi.findOldDirectory(startstamp); - - if (!olddir) - { - /* finished */ - moretodo = false; - continue; - } - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " Checking: " << olddir->path << std::endl; -#endif - - FileEntry fe; - - /* determine the full root path */ - std::string dirpath = olddir->path; - std::string rootdir = RsDirUtil::getRootDir(olddir->path); - std::string remdir = RsDirUtil::removeRootDir(olddir->path); - std::string realroot = locked_findRealRoot(rootdir); - std::string realpath = realroot; - - if (remdir != "") - realpath += "/" + remdir; - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " RealPath: " << realpath << std::endl; -#endif - - /* check for the dir existance */ - librs::util::FolderIterator dirIt(realpath); - if (!dirIt.isValid()) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << " Missing Dir: " << realpath << std::endl; - std::cerr << " Root Dir: " << rootdir << std::endl; - std::cerr << " remdir: " << remdir << std::endl; -#endif - if(directoryMap.end() != directoryMap.find(rootdir) && remdir=="") - { -#ifdef FIM_DEBUG - std::cerr << " This is a root directory. Keeping it empty." << std::endl; -#endif - } - else - { - if (!fi.removeOldDirectory(olddir->parent->path, olddir->name, stamp))/* bad directory - delete */ - { - /* bad... drop out of updateCycle() - hopefully the initial cleanup - * will deal with it next time! - otherwise we're in a continual loop - */ - std::cerr << "FileIndexMonitor::updateCycle()"; - std::cerr << "ERROR Failed to Remove: " << olddir->path << std::endl; - } - continue; - } - } - - /* update this dir - as its valid */ - fe.name = olddir->name; - fi.updateDirEntry(olddir->parent->path, fe, stamp); - - /* update the directories and files here */ - std::map::iterator dit; - std::map::iterator fit; - - /* flag existing subdirs as old */ - for(dit = olddir->subdirs.begin(); dit != olddir->subdirs.end(); ++dit) - { - fe.name = (dit->second)->name; - /* set the age as out-of-date so that it gets checked */ - fi.updateDirEntry(olddir->path, fe, 0); - } - - /* now iterate through the directory... - * directories - flags as old, - * files checked to see if they have changed. (rehashed) - */ - - to_hash.push_back(DirContentToHash()) ; - to_hash.back().realpath = realpath ; - to_hash.back().dirpath = dirpath ; - - for(;dirIt.isValid() && isRunning();dirIt.next()) - { - /* check entry type */ - std::string fname; - dirIt.d_name(fname); - std::string fullname = realpath + "/" + fname; -#ifdef FIM_DEBUG - std::cerr << "calling stats on " << fullname <path, fe, 0); - } - else if (S_ISREG(buf.st_mode)) - { - /* is file */ - bool toadd = false; -#ifdef FIM_DEBUG - std::cerr << "Is File: " << fullname << std::endl; -#endif - - fe.name = fname; - fe.size = buf.st_size; - fe.modtime = buf.st_mtime; - - /* check if it exists already */ - fit = olddir->files.find(fname); - if (fit == olddir->files.end()) - { - /* needs to be added */ -#ifdef FIM_DEBUG - std::cerr << "File Missing from List:" << fname << std::endl; -#endif - toadd = true; - } - else - { - /* check size / modtime are the same */ - // if reference_time was not inited, we revert to the old method: we test the saved mod time for the file - // versus the measured mod time. If reference is inited (this is what should always happen) we compare the measured - // mod time with the reference time. - // - if ((fe.size != (fit->second)->size) || (reference_time==0 && fe.modtime != (fit->second)->modtime) || fe.modtime > reference_time) //(fit->second)->modtime)) - { -#ifdef FIM_DEBUG - std::cerr << "File ModTime/Size changed:" << fname << std::endl; - std::cerr << "fe.modtime = " << fe.modtime << std::endl; - std::cerr << "fit.mdtime = " << fit->second->modtime << std::endl; -#endif - toadd = true; - } - else - fe.hash = (fit->second)->hash; /* keep old info */ - } - if (toadd) - { - /* push onto Hash List */ -#ifdef FIM_DEBUG - std::cerr << "Adding to Update List: "; - std::cerr << olddir->path; - std::cerr << fname << std::endl; -#endif - to_hash.back().fentries.push_back(fe); - fiMods = true ; - } - else /* update with new time */ - { -#ifdef FIM_DEBUG - std::cerr << "File Hasn't Changed:" << fname << std::endl; -#endif - fi.updateFileEntry(olddir->path, fe, stamp); - - if(cache_is_new) - hashCache.insert(realpath+"/"+fe.name,fe.size,fe.modtime,fe.hash) ; - } - } - } -#ifdef FIM_DEBUG - else - std::cout << "stat error " << errno << std::endl ; -#endif - } - - if(to_hash.back().fentries.empty()) - to_hash.pop_back() ; - - /* now we unlock the lock, and iterate through the - * next files - hashing them, before adding into the system. - */ - /* for safety - blank out data we cannot use (TODO) */ - olddir = NULL; - - /* close directory */ - dirIt.closedir(); - } - - // Now, hash all files at once. - // - if(isRunning() && !to_hash.empty()) - hashFiles(to_hash) ; - - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; - - int cleanedCount = 0; - - { /* LOCKED DIRS */ - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - /* finished update cycle - cleanup extra dirs/files that - * have not had their timestamps updated. - */ - - cleanedCount = fi.cleanOldEntries(startstamp) ; - -#ifdef FIM_DEBUG - /* print out the new directory structure */ -// fi.printFileIndex(std::cerr); -#endif - /* now if we have changed things -> restore file/hash it/and - * tell the CacheSource - */ - - if (pendingForceCacheWrite) - { - pendingForceCacheWrite = false; - fiMods = true; - } - - if (fiMods) - { - reference_time = locked_saveFileIndexes(true) ; -#ifdef FIM_DEBUG - std::cerr << "Index saved. New reference time is " << reference_time << std::endl; -#endif - } - - fi.updateHashIndex() ; // update hash map that is used to accelerate search. - fi.updateMaxModTime() ; // Update modification times for proper display. - - mInCheck = false; - - if(useHashCache) - { - hashCache.clean() ; - hashCache.save() ; - } - } - - if (cleanedCount > 0) { - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - } -} - -static std::string friendlyUnit(uint64_t val) -{ - const std::string units[5] = {"B","KB","MB","GB","TB"}; - char buf[50] ; - - double fact = 1.0 ; - - for(unsigned int i=0; i<5; ++i) - if(double(val)/fact < 1024.0) - { - sprintf(buf,"%2.2f",double(val)/fact) ; - return std::string(buf) + " " + units[i]; - } - else - fact *= 1024.0f ; - - sprintf(buf,"%2.2f",double(val)/fact*1024.0f) ; - return std::string(buf) + " TB"; -} - - -void FileIndexMonitor::hashFiles(const std::vector& to_hash) -{ - // Size interval at which we save the file lists - static const uint64_t MAX_SIZE_WITHOUT_SAVING = 10737418240ull ; // 10 GB - - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - time_t stamp = time(NULL); - - // compute total size of files to hash - uint64_t total_size = 0 ; - uint32_t n_files = 0 ; - - for(uint32_t i=0;inotifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ; - - std::string real_path = RsDirUtil::makePath(to_hash[i].realpath, fe.name); - - // 1st look into the hash cache if this file already exists. - // - if(useHashCache && hashCache.find(real_path,fe.size,fe.modtime,fe.hash)) - fi.updateFileEntry(to_hash[i].dirpath,fe,stamp); - else if(RsDirUtil::getFileHash(real_path, fe.hash,fe.size, this)) // not found, then hash it. - { - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - /* update fileIndex with new time */ - /* update with new time */ - - // Check again that the hashed file hasn't been modified since the beginning of the hashing process. - // If so, drop it. - // - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(real_path, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(real_path.c_str(), &buf)) -#endif - { - if(buf.st_mtime != fe.modtime) - std::cerr << "File " << real_path << " has been modified while being hashed. It will be dropped to avoid data inconsistency" << std::endl; - else - { - fi.updateFileEntry(to_hash[i].dirpath,fe,stamp); - - hashed_size += to_hash[i].fentries[j].size ; - - // Update the hash cache - // - if(useHashCache) - hashCache.insert(real_path,fe.size,fe.modtime,fe.hash) ; - } - } - } - else - std::cerr << "Failed to Hash File " << fe.name << std::endl; - - size += to_hash[i].fentries[j].size ; - - // don't hit the disk too hard! - usleep(10*1000); // 10 msec - - // Save the hashing result every 60 seconds, so has to save what is already hashed. -#ifdef FIM_DEBUG - std::cerr << "size - last_save_size = " << hashed_size - last_save_size << ", max=" << MAX_SIZE_WITHOUT_SAVING << std::endl ; -#endif - - if(hashed_size > last_save_size + MAX_SIZE_WITHOUT_SAVING) - { - RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_SAVE_FILE_INDEX, "") ; - - //Waiting 1 sec before pass to other hash ??is it really needed?? - usleep(1*1000*1000); // 1 sec - - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - fi.updateHashIndex() ; - FileIndexMonitor::locked_saveFileIndexes(true) ; - last_save_size = hashed_size ; - - if(useHashCache) - hashCache.save() ; - } - - // check if thread is running - running = isRunning(); - } - - fi.updateHashIndex() ; - - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - - -time_t FileIndexMonitor::locked_saveFileIndexes(bool update_cache) -{ - /* store to the cacheDirectory */ - - std::string path = getCacheDir(); - - // Multiple files are saved: for every kind of peers, the set of browsable files will be different. A specific file is - // prepared for all situations, and shared by all peers having the same situation. - // - // A complete file collection is also saved, and serves as memory for the FileIndexMonitor system. - // -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() FileIndex modified ... updating" << std::endl; -#endif - // Make for each peer the list of forbidden shared directories. Make a separate cache file for each different set. - // To figure out which sets are different, we index them by the set of forbidden indexes from the directory list. - // This is probably a bit costly, but we can't suppose that the number of shared directories is bounded. - // - std::list all_friend_ids ; - rsPeers->getFriendList(all_friend_ids); - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle(): got list of all friends." << std::endl ; - for(std::list::const_iterator it(all_friend_ids.begin());it!=all_friend_ids.end();++it) - std::cerr << " " << *it << std::endl; -#endif - - std::map, std::set > peers_per_directory_combination ; - - for(std::list::const_iterator it(all_friend_ids.begin());it!=all_friend_ids.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "About to save, with the following restrictions:" << std::endl ; - std::cerr << "Peer : " << *it << std::endl; -#endif - - std::set forbidden_dirs ; - for(std::map::const_iterator dit(directoryMap.begin());dit!=directoryMap.end();++dit) - { -#ifdef FIM_DEBUG - std::cerr << " dir=" << dit->first << ", " ; - std::cerr << "parent groups: " ; - for(std::list::const_iterator mit(dit->second.parent_groups.begin());mit!=dit->second.parent_groups.end();++mit) - std::cerr << (*mit) << ", " ; - std::cerr << std::endl;; -#endif - - FileSearchFlags permission_flags = rsPeers->computePeerPermissionFlags(*it,dit->second.shareflags,dit->second.parent_groups) ; - - if(!(permission_flags & RS_FILE_HINTS_BROWSABLE)) - { -#ifdef FIM_DEBUG - std::cerr << "forbidden" << std::endl; -#endif - forbidden_dirs.insert(dit->first) ; - } -#ifdef FIM_DEBUG - else - std::cerr << "autorized" << std::endl; -#endif - } - - peers_per_directory_combination[forbidden_dirs].insert(*it) ; - } - RsPeerId ownId = rsPeers->getOwnId() ; - peers_per_directory_combination[std::set()].insert(ownId) ; // add full configuration to self, i.e. no forbidden directories. - - int n=0 ; - time_t now = time(NULL) ; - time_t mod_time = 0 ; - - for(std::map, std::set >::const_iterator it(peers_per_directory_combination.begin()); - it!=peers_per_directory_combination.end();++it,++n) - { - std::string tmpname_browsable; - - if(it->first.empty()) - rs_sprintf(tmpname_browsable, "fc-own-%ld.rsfc",now,n); - else - rs_sprintf(tmpname_browsable, "fc-own-%ld.%04d",now,n); - - std::string fname_browsable = path + "/" + tmpname_browsable; - -#ifdef FIM_DEBUG - std::cerr << "Sending file list: " << std::endl; - std::cerr << " filename : " << tmpname_browsable << std::endl; - std::cerr << " to peers : " << std::endl; - for(std::set::const_iterator itt(it->second.begin());itt!= it->second.end();++itt) - std::cerr << " " << *itt << std::endl; - std::cerr << " forbidden : " << std::endl; - for(std::set::const_iterator itt(it->first.begin());itt!= it->first.end();++itt) - std::cerr << " " << *itt << std::endl; -#endif - - RsFileHash hash ; - uint64_t size ; - -#ifdef FIM_DEBUG - std::cerr << "writing file " << fname_browsable << std::endl; -#endif - fi.saveIndex(fname_browsable, hash, size,it->first); // save only browsable files - - if(size > 0) - { -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() saved with hash:" << hash << std::endl; -#endif - - /* should clean up the previous cache.... */ - - /* flag as new info */ - RsCacheData data; - data.pid = fi.root->id; - data.cid.type = getCacheType(); - data.cid.subid = n; - data.path = path; - data.name = tmpname_browsable; - data.hash = hash; - data.size = size; - data.recvd = time(NULL); - - for(std::set::const_iterator ff(it->second.begin());ff!=it->second.end();++ff) - _cache_items_per_peer[*ff] = data ; - - data.cid.subid = n; - - if(update_cache) - updateCache(data,it->second); - } - - if(it->first.empty()) - { - // Computes the reference time. - // - struct stat64 buf; - -#ifdef WINDOWS_SYS - std::wstring wfullname; - librs::util::ConvertUtf8ToUtf16(fname_browsable, wfullname); - if ( 0 == _wstati64(wfullname.c_str(), &buf)) -#else - if ( 0 == stat64(fname_browsable.c_str(), &buf)) -#endif - { - mod_time = buf.st_mtime ; - } - else - { - std::cerr << "(EE) Error. Cannot get the proper modification time for file " << fname_browsable << " errno=" << errno << std::endl; - mod_time = 0 ; - } - } - } - -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::updateCycle() called updateCache()"; - std::cerr << std::endl; -#endif - return mod_time ; -} - -bool FileIndexMonitor::cachesAvailable(const RsPeerId &pid,std::map &ids) -{ - lockData() ; -#ifdef FIM_DEBUG - std::cerr << "In cachesAvailable..." << std::endl; -#endif - - // Go through the list of saved cache items for that particular peer. - // - ids.clear() ; - std::map::const_iterator it(_cache_items_per_peer.find(pid)) ; - RsPeerId ownId = rsPeers->getOwnId(); - - if(it != _cache_items_per_peer.end()) - { - ids[it->second.cid] = it->second ; - - if(pid != ownId) - ids[it->second.cid].cid.subid = 0 ; // Force subid to be 0, so that it's - // not going to be mixed up at the client with other files received if the - // subid changes for that peer. - // -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor: caches available for peer " << pid << ": " << it->second.name << std::endl ; -#endif - } -#ifdef FIM_DEBUG - else - std::cerr << "No cache item for peer " << pid << std::endl; -#endif - - unlockData() ; - - return true ; -} - -void FileIndexMonitor::updateShareFlags(const SharedDirInfo& dir) -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - bool fimods = false ; -#ifdef FIM_DEBUG - std::cerr << "*** FileIndexMonitor: Updating flags for " << dir.filename << " to " << dir.shareflags << std::endl ; -#endif - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - for(std::list::iterator it(pendingDirList.begin());it!=pendingDirList.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "** testing pending dir " << (*it).filename << std::endl ; -#endif - if((*it).filename == dir.filename) - { -#ifdef FIM_DEBUG - std::cerr << "** Updating to " << (*it).shareflags << "!!" << std::endl ; -#endif - (*it).shareflags = dir.shareflags ; - (*it).parent_groups = dir.parent_groups ; - break ; - } - } - - for(std::map::iterator it(directoryMap.begin());it!=directoryMap.end();++it) - { -#ifdef FIM_DEBUG - std::cerr << "** testing " << (*it).second.filename << std::endl ; -#endif - if((*it).second.filename == dir.filename) - { -#ifdef FIM_DEBUG - std::cerr << "** Updating from " << it->second.shareflags << "!!" << std::endl ; -#endif - (*it).second.shareflags = dir.shareflags ; - (*it).second.parent_groups = dir.parent_groups ; - fimods = true ; - break ; - } - } - } - if(fimods) - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - locked_saveFileIndexes(true) ; - } - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - /* interface */ -void FileIndexMonitor::setSharedDirectories(const std::list& dirs) -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - - std::list checkeddirs; - - std::list::const_iterator it; -#ifdef FIM_DEBUG - std::cerr << "FileIndexMonitor::setSharedDirectories() :\n"; -#endif - - for(it = dirs.begin(); it != dirs.end(); ++it) - { - -#ifdef FIM_DEBUG - std::cerr << "\t" << (*it).filename; - std::cerr << std::endl; -#endif - - checkeddirs.push_back(*it); - -#ifdef REMOVED_CODE - // this code has been removed because it prevents unmounted shared directories to stay in the list of shared files. It's better - // to keep them showing empty than removing them completely. - // - /* check if dir exists before adding in */ -// std::string path = (*it).filename; -// if (!RsDirUtil::checkDirectory(path)) -// { -//#ifdef FIM_DEBUG -// std::cerr << "FileIndexMonitor::setSharedDirectories()"; -// std::cerr << " Ignoring NonExistant SharedDir: " << path << std::endl; -//#endif -// } -// else -// { -// checkeddirs.push_back(*it); -// } -#endif - } - - { - RsStackMutex stack(fiMutex) ;/* LOCKED DIRS */ - - pendingDirs = true; - pendingDirList = checkeddirs; - } - - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); -} - - /* interface */ -void FileIndexMonitor::getSharedDirectories(std::list &dirs) -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - /* must provide pendingDirs, as other parts depend on instanteous response */ - if (pendingDirs) - dirs = pendingDirList; - else - { - /* get actual list (not pending stuff) */ - std::map::const_iterator it; - - for(it = directoryMap.begin(); it != directoryMap.end(); ++it) - dirs.push_back(it->second) ; - } -} - - - /* interface */ -void FileIndexMonitor::forceDirectoryCheck() -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - if (!mInCheck) - mForceCheck = true; -} - - -void FileIndexMonitor::forceDirListsRebuildAndSend() -{ - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - locked_saveFileIndexes(true) ; -} - - /* interface */ -bool FileIndexMonitor::inDirectoryCheck() -{ - RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ - - return mInCheck; -} - - -bool FileIndexMonitor::internal_setSharedDirectories() -{ - bool changed = false; - - { - RsStackMutex stack(fiMutex) ; /* LOCKED DIRS */ - - if (!pendingDirs) - { - if (mForceCheck) - { - mForceCheck = false; - return true; - } - - return false; - } - - mForceCheck = false; - pendingDirs = false; - pendingForceCacheWrite = true; - - /* clear old directories */ - directoryMap.clear(); - - /* iterate through the directories */ - std::list::iterator it; - std::map::const_iterator cit; - for(it = pendingDirList.begin(); it != pendingDirList.end(); ++it) - { - /* get the head directory */ - std::string root_dir = (*it).filename; - std::string top_dir = it->virtualname; - if (top_dir.empty()) { - top_dir = RsDirUtil::getTopDir(root_dir); - } - - /* if unique -> add, else add modifier */ - bool unique = false; - for(int i = 0; !unique; ++i) - { - std::string tst_dir = top_dir; - if (i > 0) - { - rs_sprintf_append(tst_dir, "-%d", i); - } - if (directoryMap.end()== (cit=directoryMap.find(tst_dir))) - { - unique = true; - /* store calculated name */ - it->virtualname = tst_dir; - /* add it! */ - directoryMap[tst_dir.c_str()] = *it; -#ifdef FIM_DEBUG - std::cerr << "Added [" << tst_dir << "] => " << root_dir << std::endl; -#endif - } - } - } - - pendingDirList.clear(); - - /* now we've decided on the 'root' dirs set them to the - * fileIndex - */ - std::list topdirs; - for(cit = directoryMap.begin(); cit != directoryMap.end(); ++cit) - { - topdirs.push_back(cit->first); - } - - if (fi.setRootDirectories(topdirs, 0) > 0) - { - changed = true; - } - - locked_saveFileIndexes(true) ; - } - - if (changed) - { - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0); - } - - return true; -} - -/* lookup directory function */ -std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const -{ - /**** MUST ALREADY BE LOCKED ****/ - std::string realroot = ""; - - std::map::const_iterator cit; - if (directoryMap.end()== (cit=directoryMap.find(rootdir))) - { - std::cerr << "FileIndexMonitor::locked_findRealRoot() Invalid RootDir: "; - std::cerr << rootdir << std::endl; - } - else - { - realroot = cit->second.filename; - } - - return realroot; -} - -int FileIndexMonitor::RequestDirDetails(const std::string& path, DirDetails& details) const -{ - /* lock it up */ - RsStackMutex mutex(fiMutex) ; - return fi.extractData(path,details) ; -} - -uint32_t FileIndexMonitor::getType(void *ref) const -{ - RsStackMutex mutex(fiMutex) ; - - return fi.getType(ref) ; -} -int FileIndexMonitor::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const -{ - /* remove unused parameter warnings */ - (void) flags; - - RsStackMutex mutex(fiMutex) ; - -#ifdef FIM_DEBUG2 - std::cerr << "FileIndexMonitor::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl; -#endif - - /* root case */ - -#ifdef FIM_DEBUG2 - fi.root->checkParentPointers(); -#endif - - // If ref is NULL, we build a root node - - if (ref == NULL) - { -#ifdef FI_DEBUG2 - std::cerr << "FileIndex::RequestDirDetails() ref=NULL (root)" << std::endl; -#endif - /* local only */ - DirStub stub; - stub.type = DIR_TYPE_PERSON; - stub.name = fi.root->name; - stub.ref = fi.root; - details.children.push_back(stub); - details.count = 1; - - details.parent = NULL; - details.prow = -1; - details.ref = NULL; - details.type = DIR_TYPE_ROOT; - details.name = "root"; - details.hash.clear() ; - details.path = "root"; - details.age = 0; - details.flags.clear() ; - details.min_age = 0 ; - - return true ; - } - - bool b = FileIndex::extractData(ref,details) ; - - if(!b) - return false ; - - // look for the top level and setup flags accordingly - // The top level directory is the first dir in parents for which - // dir->parent->parent == NULL - - if(ref != NULL) - { - FileEntry *file = (FileEntry *) ref; - locked_findShareFlagsAndParentGroups(file,details.flags,details.parent_groups) ; - } - return true ; -} - -void FileIndexMonitor::locked_findShareFlagsAndParentGroups(FileEntry *file,FileStorageFlags& flags,std::list& parent_groups) const -{ - flags.clear() ; - static const FileStorageFlags PERMISSION_MASK = DIR_FLAGS_BROWSABLE_OTHERS | DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_GROUPS | DIR_FLAGS_NETWORK_WIDE_GROUPS ; - - DirEntry *dir = dynamic_cast(file) ; - if(dir == NULL) - dir = dynamic_cast(file->parent) ; - - if(dir != NULL && dir->parent != NULL) - while(dir->parent->parent != NULL) - dir = dir->parent ; - - if(dir != NULL && dir->parent != NULL) - { -#ifdef FIM_DEBUG2 - std::cerr << "FileIndexMonitor::RequestDirDetails: top parent name=" << dir->name << std::endl ; -#endif - std::map::const_iterator it = directoryMap.find(dir->name) ; - - if(it == directoryMap.end()) - std::cerr << "*********** ERROR *********** In " << __PRETTY_FUNCTION__ << std::endl ; - else - { - flags = it->second.shareflags ; - flags &= PERMISSION_MASK ; - flags &= ~DIR_FLAGS_NETWORK_WIDE_GROUPS ; // Disabling this flag for now, because it has inconsistent effects. - parent_groups = it->second.parent_groups ; - } -#ifdef FIM_DEBUG2 - std::cerr << "flags = " << flags << std::endl ; -#endif - } -} - - diff --git a/libretroshare/src/dbase/fimonitor.h b/libretroshare/src/dbase/fimonitor.h deleted file mode 100644 index 2cdddb4af..000000000 --- a/libretroshare/src/dbase/fimonitor.h +++ /dev/null @@ -1,220 +0,0 @@ -/* - * RetroShare FileCache Module: fimonitor.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef FILE_INDEX_MONITOR_H -#define FILE_INDEX_MONITOR_H - -#include "dbase/cachestrapper.h" -#include "dbase/findex.h" -#include "util/rsthreads.h" -#include "retroshare/rsfiles.h" - -/****************************************************************************************** - * The Local Monitoring Class: FileIndexMonitor. - * - * This periodically scans the directory tree, and updates any modified directories/files. - * - *****************************************************************************************/ - -/****************************************************************************************** - STILL TODO: - - (1) Implement Hash function. - -bool FileIndexMonitor::hashFile(std::string path, FileEntry &fi); - - (2) Add Shared directory controls to Monitor. - -int FileIndexMonitor::addSharedDirectory(std::path); -int FileIndexMonitor::removeSharedDirectory(std::path); -std::string FileIndexMonitor::findRealRoot(std::string base); - - These must be split into / and the mapping saved. - eg: addSharedDirectory("c:/home/stuff/dir1") --> "c:/home/stuff" <-> "dir1" - This code has been written already, and can just be moved over. - - FOR LATER: - (2) Port File/Directory lookup code to windoze. (or compile dirent.c under windoze) - (3) Add Load/Store interface to FileIndexMonitor. (later) - (4) Integrate with real Thread/Mutex code (last thing to do) - -******************************************************************************************/ - - -class DirContentToHash -{ - public: - std::vector fentries ; - - std::string realpath ; - std::string dirpath ; -}; - -class HashCache -{ - public: - HashCache(const std::string& save_file_name) ; - - void save() ; - void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ; - bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ; - void clean() ; - - typedef struct - { - uint64_t size ; - uint64_t time_stamp ; - uint64_t modf_stamp ; - RsFileHash hash ; - } HashCacheInfo ; - - void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; } - uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; } - void clear() { _files.clear(); } - bool empty() const { return _files.empty() ; } - private: - uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries - std::map _files ; - std::string _path ; - bool _changed ; -}; - -/****************************************************************************************** - * FileIndexMonitor - *****************************************************************************************/ - -class FileIndexMonitor: public CacheSource, public RsTickingThread -{ - public: - FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid, const std::string& config_dir); - virtual ~FileIndexMonitor(); - - /* external interface for filetransfer */ - bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list& parent_groups) const; - - int SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) ; - int SearchBoolExp(Expression *exp, std::list &results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - - int filterResults(std::list& firesults,std::list& results,FileSearchFlags flags,const RsPeerId& peer_id) const ; - - - /* external interface for local access to files */ - bool convertSharedFilePath(std::string path, std::string &fullpath); - - - /* Interacting with CacheSource */ - /* overloaded from CacheSource */ - virtual bool loadLocalCache(const RsCacheData &data); /* called with stored data */ - bool updateCache(const RsCacheData &data,const std::set& dest_peers); /* we call when we have a new cache for others */ - - - /* the FileIndexMonitor inner workings */ - //virtual void run(std::string& currentJob); /* overloaded from RsThread */ - //void updateCycle(std::string& currentJob); - virtual void data_tick(); /* overloaded from RsThread */ - void updateCycle(); - - // Interface for browsing dir hirarchy - int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ; - uint32_t getType(void*) const ; - int RequestDirDetails(const std::string& path, DirDetails &details) const ; - - // set/update shared directories - virtual void setSharedDirectories(const std::list& dirs); - void getSharedDirectories(std::list& dirs); - void updateShareFlags(const SharedDirInfo& info) ; - - void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed - void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files. - bool inDirectoryCheck(); - - /* util fns */ - - // from CacheSource - virtual bool cachesAvailable(const RsPeerId& pid, std::map &ids) ; - - protected: - // Sets/gets the duration period within which already hashed files are remembered. - // - void setRememberHashFilesDuration(uint32_t days) ; - uint32_t rememberHashFilesDuration() const ; - void setRememberHashFiles(bool) ; - bool rememberHashFiles() ; - // Remove any memory of formerly hashed files that are not shared anymore - void clearHashFiles() ; - void setPeriod(int insecs); - int getPeriod() const; - - bool autoCheckEnabled() const ; - - private: - /* the mutex should be locked before calling these 3. */ - - // Saves file indexs and update the cache. Returns the name of the main - // file index, which becomes the new reference file for mod times. - // - time_t locked_saveFileIndexes(bool update_cache) ; - - // Finds the share flags associated with this file entry. - void locked_findShareFlagsAndParentGroups(FileEntry *fe, FileStorageFlags& shareflags, std::list &parent_groups) const ; - - std::string locked_findRealRoot(std::string base) const; - - void hashFiles(const std::vector& to_hash) ; - bool hashFile(std::string path, FileEntry &fi); /* To Implement */ - - /* data */ - - mutable RsMutex fiMutex; - - FileIndex fi; - - int updatePeriod; - std::map directoryMap; /* used by findRealRoot */ - - /* flags to kick - if we were busy or sleeping */ - bool pendingDirs; - bool pendingForceCacheWrite; - - /* flags to force Check, to tell if we're in check */ - bool mForceCheck; - bool mInCheck; - - std::list pendingDirList; - bool internal_setSharedDirectories(); - - HashCache hashCache ; - bool useHashCache ; - - std::map _cache_items_per_peer ; // stored the cache items to be sent to each peer. - - // This file is the location of the current index file. When checking for new files, we compare the modification time - // of this file to the mod time of the files on the disk. This allows to now account for time-shift in the computer. - // - time_t reference_time ; -}; - - -#endif - - diff --git a/libretroshare/src/dbase/findex.cc b/libretroshare/src/dbase/findex.cc deleted file mode 100644 index 17b5dba0b..000000000 --- a/libretroshare/src/dbase/findex.cc +++ /dev/null @@ -1,1516 +0,0 @@ -/* - * RetroShare FileCache Module: findex.cc - * - * Copyright 2004-2007 by Robert Fernie, Kefei Zhou. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include -#include -#include "dbase/findex.h" -#include "retroshare/rsexpr.h" -#include "util/rsdir.h" -#include "util/rsstring.h" -#include "util/rscompress.h" - -#include -#include -#include -#include -#ifdef __MACH__ -#include -#else -#include -#endif -#include -#include -#include -#include -#include - -#include -#include - -// This char is used to separate fields in the file list cache. It is supposed to be -// sufficiently safe on all systems. -// -static const char FILE_CACHE_SEPARATOR_CHAR = '|' ; - -/**** - * #define FI_DEBUG 1 - * #define FI_DEBUG_ALL 1 - ****/ - -static RsMutex FIndexPtrMtx("FIndexPtrMtx") ; -#ifdef __MACH__ -std::unordered_set FileIndex::_pointers ; -#else -std::tr1::unordered_set FileIndex::_pointers ; -#endif -void FileIndex::registerEntry(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - _pointers.insert(p) ; -} -void FileIndex::unregisterEntry(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - _pointers.erase(p) ; -} -bool FileIndex::isValid(void*p) -{ - RsStackMutex m(FIndexPtrMtx) ; - return _pointers.find(p) != _pointers.end() ; -} - -DirEntry::~DirEntry() -{ - /* cleanup */ - std::map::iterator dit; - std::map::iterator fit; - - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { - FileIndex::unregisterEntry((void*)dit->second) ; - delete (dit->second); - } - subdirs.clear(); - - for(fit = files.begin(); fit != files.end(); ++fit) - { - FileIndex::unregisterEntry((void*)fit->second) ; - delete (fit->second); - } - files.clear(); -} - - -int DirEntry::checkParentPointers() -{ -#ifdef FI_DEBUG - updateChildRows(); - std::map::iterator dit; - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { - /* debug check */ - (dit->second)->checkParentPointers(); - } -#endif - return 1; -} - - - -int DirEntry::updateChildRows() -{ - /* iterate through children and set row (parent should be good) */ - std::map::iterator dit; - std::map::iterator fit; - int i = 0; - for(dit = subdirs.begin(); dit != subdirs.end(); ++dit) - { -#ifdef FI_DEBUG - /* debug check */ - if ((dit->second)->parent != this) - { - std::cerr << "DirEntry::updateChildRows()"; - std::cerr << "****WARNING subdir Parent pointer invalid!"; - std::cerr << std::endl; - (dit->second)->parent = this; - } -#endif - (dit->second)->row = i++; - } - - for(fit = files.begin(); fit != files.end(); ++fit) - { -#ifdef FI_DEBUG - /* debug check */ - if ((fit->second)->parent != this) - { - std::cerr << "DirEntry::updateChildRows()"; - std::cerr << "****WARNING file Parent pointer invalid!"; - std::cerr << std::endl; - (fit->second)->parent = this; - } -#endif - (fit->second)->row = i++; - } - return 1; -} - - -int DirEntry::removeDir(const std::string& name) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() != (it = subdirs.find(name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeDir() Cleaning up dir: " << name; - std::cerr << std::endl; -#endif - ndir = (it->second); - - subdirs.erase(it); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - /* update row counters */ - updateChildRows(); - return 1; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeDir() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - -int DirEntry::removeFile(const std::string& name) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - FileEntry *nfile = NULL; - - if (files.end() != (it = files.find(name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeFile() removing File: " << name; - std::cerr << std::endl; -#endif - nfile = (it->second); - - files.erase(it); - FileIndex::unregisterEntry((void*)nfile) ; - delete nfile; - /* update row counters */ - updateChildRows(); - return 1; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeFile() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - - - -int DirEntry::removeOldDir(const std::string& name, time_t old) -{ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() != (it = subdirs.find(name))) - { - ndir = (it->second); - if (ndir->updtime < old) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeOldDir() Removing Old dir: " << name; - std::cerr << std::endl; -#endif - subdirs.erase(it); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - - /* update row counters */ - updateChildRows(); - return 1; - } -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::removeOldDir() Keeping UptoDate dir: " << name; - std::cerr << std::endl; -#endif - return 0; - } - -#ifdef FI_DEBUG - std::cerr << "DirEntry::removeDir() missing Entry: " << name; - std::cerr << std::endl; -#endif - return 0; -} - - - - -int DirEntry::removeOldEntries(time_t old, bool recursive) -{ - /* remove old dirs from our lists -> then do children and files */ - - /* get all dirs with old time */ - std::list removeList; - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - if ((it->second)->updtime < old) - { - removeList.push_back(it->second); - } - } - - int count = removeList.size(); - - /* now remove the old entries */ - std::list::iterator rit; - for(rit = removeList.begin(); rit != removeList.end(); ++rit) - { - removeDir((*rit)->name); - } - - if (recursive) - { - /* now handle children */ - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - count += (it->second)->removeOldEntries(old, recursive); - } - } - - /* now handle files similarly */ - std::list removeFileList; - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - if ((fit->second)->updtime < old) - { - removeFileList.push_back(fit->second); - } - } - - count += removeFileList.size(); - - /* now remove the old entries */ - std::list::iterator rfit; - for(rfit = removeFileList.begin(); rfit != removeFileList.end(); ++rfit) - { - removeFile((*rfit)->name); - } - - return count; -} - - -DirEntry *DirEntry::findOldDirectory(time_t old) -{ - /* check if one of our directories is old ... - */ - - /* get all dirs with old time */ - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - if ((it->second)->updtime < old) - { - return (it->second); - } - } - - /* - * else check chlidren. - */ - - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - DirEntry *olddir = (it->second)->findOldDirectory(old); - if (olddir) - { - return olddir; - } - } - - return NULL; -} - - -DirEntry *DirEntry::findDirectory(const std::string& fpath) -{ - std::string nextdir = RsDirUtil::getRootDir(fpath); - std::map::iterator it; - if (subdirs.end() == (it = subdirs.find(nextdir))) - { -#ifdef FI_DEBUG - std::cerr << "DirEntry::findDirectory() Missing subdir:"; - std::cerr << "\"" << nextdir << "\""; - std::cerr << std::endl; -#endif - return NULL; - } - - std::string rempath = RsDirUtil::removeRootDir(fpath); - if (rempath == "") - { - return it->second; - } - - - // Adding more lenient directory look up. - // returns lower directory to fpath is a FILE. - DirEntry *subdir = (it->second)->findDirectory(rempath); - if (subdir) - return subdir; - else - return it->second; -} - - -int DirEntry::updateDirectories(const std::string& fpath, int new_pop, int new_modtime) -{ - int ret = 1; - if (path != "") /* if not there -> continue down tree */ - { - std::string nextdir = RsDirUtil::getRootDir(fpath); - std::map::iterator it; - if (subdirs.end() == (it = subdirs.find(nextdir))) - { - return 0; - } - - std::string rempath = RsDirUtil::removeRootDir(fpath); - ret = (it->second)->updateDirectories(rempath, new_pop, new_modtime); - } - - if (ret) /* if full path is okay -> update and return ok */ - { - /* this is assumes that pop always increases! */ - if (new_pop > pop) - { - pop = new_pop; - } - if (new_modtime > modtime) - { - modtime = new_modtime; - } - } - return ret; -} - -DirEntry *DirEntry::updateDir(const FileEntry& fe, time_t utime) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - DirEntry *ndir = NULL; - if (subdirs.end() == (it = subdirs.find(fe.name))) - { -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateDir() Adding Entry"; - std::cerr << std::endl; -#endif - ndir = new DirEntry(); - FileIndex::registerEntry((void*)ndir) ; - ndir -> parent = this; - ndir -> path = path + "/" + fe.name; - ndir -> name = fe.name; - ndir -> pop = 0; - ndir -> modtime = 0; - ndir -> updtime = utime; - - subdirs[fe.name] = ndir; - - /* update row counters */ - updateChildRows(); - return ndir; - } - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateDir() Updating Entry"; - std::cerr << std::endl; -#endif - - /* update utime */ - ndir = (it->second); - ndir->updtime = utime; - - return ndir; -} - - -FileEntry *DirEntry::updateFile(const FileEntry& fe, time_t utime) -{ - /* if it doesn't exist - add */ - std::map::iterator it; - FileEntry *nfile = NULL; - if (files.end() == (it = files.find(fe.name))) - { - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateFile() Adding Entry"; - std::cerr << std::endl; -#endif - - nfile = new FileEntry(); - FileIndex::registerEntry((void*)nfile) ; - nfile -> parent = this; - nfile -> name = fe.name; - nfile -> hash = fe.hash; - nfile -> size = fe.size; - nfile -> pop = 0; - nfile -> modtime = fe.modtime; - nfile -> updtime = utime; - - files[fe.name] = nfile; - - /* update row counters */ - updateChildRows(); - return nfile; - } - - -#ifdef FI_DEBUG_ALL - std::cerr << "DirEntry::updateFile() Updating Entry"; - std::cerr << std::endl; -#endif - - - /* update utime */ - nfile = (it->second); - nfile -> parent = this; - nfile -> name = fe.name; - nfile -> hash = fe.hash; - nfile -> size = fe.size; - nfile -> modtime = fe.modtime; - nfile -> updtime = utime; - //nfile -> pop = 0; // not handled here. - - return nfile; -} - - -int FileEntry::print(std::string &out) -{ - /* print this dir, then subdirs, then files */ - - rs_sprintf_append(out, "file %03d [%ld/%ld] : ", row, updtime, modtime); - - if (parent) - out += parent->path; - else - out += "[MISSING PARENT]"; - - rs_sprintf_append(out, " %s [ s: %lld ] ==> [ %s ]\n", name.c_str(), size, hash.toStdString().c_str()); - - return 1; -} - - -int DirEntry::print(std::string &out) -{ - /* print this dir, then subdirs, then files */ - rs_sprintf_append(out, "dir %03d [%ld] : %s\n", row, updtime, path.c_str()); - - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - (it->second)->print(out); - } - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - (fit->second)->print(out); - } - return 1; -} - -FileIndex::FileIndex(const RsPeerId& pid) -{ - root = new PersonEntry(pid); - registerEntry(root) ; - _file_hashes.clear() ; -} - -FileIndex::~FileIndex() -{ - FileIndex::unregisterEntry((void*)root) ; - delete root; -} - -int FileIndex::setRootDirectories(const std::list &inlist, time_t updtime) -{ - /* set update time to zero */ - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - (it->second)->updtime = 0; - } - - std::list::const_iterator ait; - FileEntry fe; - time_t utime = 1; - for(ait = inlist.begin(); ait != inlist.end(); ++ait) - { - fe.name = (*ait); - - /* see if it exists */ - root->updateDir(fe, utime); - } - - /* remove all dirs with zero time (non recursive) */ - int cleanedCount = root->removeOldEntries(utime, false); - - /* now flag remaining directories with correct update time */ - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - (it->second)->updtime = updtime; - } - - // update file hash index. - - updateHashIndex() ; - - return cleanedCount; -} - -void FileIndex::updateHashIndex() -{ - _file_hashes.clear() ; - recursUpdateHashIndex(root) ; -} - -void FileIndex::recursUpdateHashIndex(DirEntry *dir) -{ - for(std::map::iterator it(dir->subdirs.begin());it!=dir->subdirs.end();++it) - recursUpdateHashIndex(it->second) ; - - for(std::map::iterator it(dir->files.begin());it!=dir->files.end();++it) - _file_hashes[it->second->hash] = it->second ; -} - -void FileIndex::updateMaxModTime() -{ - RecursUpdateMaxModTime(root) ; -} -void FileIndex::RecursUpdateMaxModTime(DirEntry *dir) -{ - time_t max_mod_t = 0 ; - - for(std::map::iterator it(dir->subdirs.begin());it!=dir->subdirs.end();++it) - { - RecursUpdateMaxModTime(it->second) ; - max_mod_t = std::max(max_mod_t, it->second->most_recent_time) ; - } - for(std::map::iterator it(dir->files.begin());it!=dir->files.end();++it) - max_mod_t = std::max(max_mod_t, it->second->modtime) ; - - dir->most_recent_time = max_mod_t ; -} - -int FileIndex::getRootDirectories(std::list &outlist) -{ - /* set update time to zero */ - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - outlist.push_back(it->first); - } - return 1; -} - -/* update (index building) */ -DirEntry *FileIndex::updateDirEntry(const std::string& fpath, const FileEntry& fe, time_t utime) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::updateDirEntry() Path: \""; - std::cerr << fpath << "\"" << " + \"" << fe.name << "\""; - std::cerr << std::endl; -#endif - DirEntry *parent = NULL; - if (fpath == "") - { - parent = root; - } - else - { - parent = root->findDirectory(fpath); - } - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::updateDirEntry() NULL parent"; - std::cerr << std::endl; -#endif - return NULL; - } - return parent -> updateDir(fe, utime); -} - - -FileEntry *FileIndex::updateFileEntry(const std::string& fpath, const FileEntry& fe, time_t utime) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::updateFileEntry() Path: \""; - std::cerr << fpath << "\"" << " + \"" << fe.name << "\""; - std::cerr << std::endl; -#endif - DirEntry *parent = root->findDirectory(fpath); - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::updateFileEntry() NULL parent"; - std::cerr << std::endl; -#endif - return NULL; - } - - return parent -> updateFile(fe, utime); -} - - -DirEntry *FileIndex::findOldDirectory(time_t old) /* finds directories older than old */ -{ - DirEntry *olddir = root->findOldDirectory(old); -#ifdef FI_DEBUG - - std::cerr << "FileIndex::findOldDirectory(" << old << ") -> "; - if (olddir) - { - std::cerr << olddir->path; - } - else - { - std::cerr << "NONE"; - } - std::cerr << std::endl; - -#endif - return olddir; -} - -int FileIndex::removeOldDirectory(const std::string& fpath, const std::string& name, time_t old) -{ - /* path is to parent */ -#ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::removeOldDirectory() Path: \""; - std::cerr << fpath << "\"" << " + \"" << name << "\""; - std::cerr << std::endl; -#endif - - /* because of this find - we cannot get a child of - * root (which is what we want!) - */ - - DirEntry *parent = root->findDirectory(fpath); - /* for root directory case ... no subdir. */ - if (fpath == "") - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::removeOldDirectory() removing a root dir"; - std::cerr << std::endl; -#endif - parent = root; - } - - if (!parent) { -#ifdef FI_DEBUG - std::cerr << "FileIndex::removeOldDirectory() NULL parent"; - std::cerr << std::endl; -#endif - return 0; - } - return parent -> removeOldDir(name, old); -} - - -int FileIndex::cleanOldEntries(time_t old) /* removes entries older than old */ -{ - int count = 0; - - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { - count += (it->second)->removeOldEntries(old, true); - } - return count; -} - - - -int FileIndex::printFileIndex(std::ostream &out) -{ - std::string sout ; - printFileIndex(sout) ; - out << sout << std::endl; - - return 1 ; -} -int FileIndex::printFileIndex(std::string &out) -{ - out += "FileIndex::printFileIndex()\n"; - root->print(out); - return 1; -} - -int FileIndex::loadIndex(const std::string& filename, const RsFileHash& expectedHash, uint64_t size) -{ - FILE *file = RsDirUtil::rs_fopen(filename.c_str(),"rb") ; - - if (!file) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::loadIndex error opening file: " << filename; - std::cerr << std::endl; -#endif - return 0; - } - - std::string s ; - - { - /* load file into memory, close file */ - RsTemporaryMemory compressed_data(size) ; - - if(!compressed_data) - { - std::cerr << "FileIndex::loadIndex(): can't allocate memory for " << size << " bytes." << std::endl; - fclose(file); - return 0; - } - uint64_t bytesread = 0 ; - if(size != (bytesread = fread(compressed_data,1,size,file))) - { - std::cerr << "FileIndex::loadIndex(): can't read " << size << " bytes from file " << filename << ". Only " << bytesread << " actually read." << std::endl; - fclose(file); - return 0; - } - fclose(file); - - RsFileHash tmpout = RsDirUtil::sha1sum((unsigned char *)(compressed_data),size); - - // /* calculate hash */ - // unsigned char sha_buf[SHA_DIGEST_LENGTH]; - // SHA_CTX *sha_ctx = new SHA_CTX; - // SHA1_Init(sha_ctx); - // SHA1_Update(sha_ctx, s.c_str(), s.length()); - // SHA1_Final(&sha_buf[0], sha_ctx); - // delete sha_ctx; - // - // std::string tmpout; - // for(int i = 0; i < SHA_DIGEST_LENGTH; ++i) - // { - // rs_sprintf_append(tmpout, "%02x", (unsigned int) (sha_buf[i])); - // } - - if (!expectedHash.isNull() && expectedHash != tmpout) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::loadIndex expected hash does not match" << std::endl; - std::cerr << "Expected hash: " << expectedHash << std::endl; - std::cerr << "Hash found: " << tmpout << std::endl; -#endif - return 0; - } - // now uncompress the string - // - - uint8_t *uncompressed_data = NULL ; - unsigned int uncompressed_data_size = 0 ; - - if(!RsCompress::uncompress_memory_chunk(compressed_data,size,uncompressed_data,uncompressed_data_size)) - { - std::cerr << "FileIndex::loadIndex() Decompression failed! Fileindex can't be read." << std::endl; - free(uncompressed_data); - return 0 ; - } - s = std::string((char *)uncompressed_data,uncompressed_data_size) ; - - std::cerr << " file = " << filename << std::endl; - std::cerr << " uncompressed size = " << uncompressed_data_size << std::endl; - std::cerr << " compressed size = " << size << std::endl; - std::cerr << " hash = " << tmpout << std::endl; - - free(uncompressed_data) ; - } - -#define FIND_NEXT(s,start,end,c) end = s.find(c, start); if (end == std::string::npos) end = s.length(); - - DirEntry *ndir = NULL; - FileEntry *nfile = NULL; - std::list dirlist; - std::string word; - char ch; - - std::string::size_type pos = 0; - while (pos < s.length()) - { - ch = s[pos]; - ++pos; - if (ch == '-') - { - FIND_NEXT(s, pos, pos, '\n'); - ++pos; - - switch(dirlist.size()) - { - /* parse error: out of directory */ - case 0: - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " Ran out of dirs"; - std::cerr << std::endl; -#endif - goto error; - } - /* finished parse, last dir is root */ - case 1: - { - RsPeerId pid = root -> id; - FileIndex::unregisterEntry((void*)root) ; - delete root; /* to clean up old entries */ - root = new PersonEntry(pid); - registerEntry((void*)root) ; - - /* shallow copy of all except id */ - ndir = dirlist.back(); - dirlist.pop_back(); /* empty list */ - (*root) = (*ndir); - - /* now cleanup (can't call standard delete) */ - ndir->subdirs.clear(); - ndir->files.clear(); - FileIndex::unregisterEntry((void*)ndir) ; - delete ndir; - ndir = NULL; - - /* must reset parent pointers now */ - std::map::iterator it; - for(it = root->subdirs.begin(); - it != root->subdirs.end(); ++it) - { - (it->second)->parent = root; - } - - break; - } - /* pop stack */ - default: dirlist.pop_back(); ndir = dirlist.back(); - } - continue; - } - - // Ignore comments - else if (ch == '#') - { - FIND_NEXT(s, pos, pos, '\n'); - ++pos; - } - - else { - std::vector tokens; - /* parse line */ - std::string::size_type lineend; - FIND_NEXT(s, pos, lineend, '\n'); - std::string line = s.substr(pos, lineend - pos); - pos = lineend + 1; - - std::string::size_type start = 0; - while (start < line.length()) - { - std::string::size_type end; - FIND_NEXT(line, start, end, FILE_CACHE_SEPARATOR_CHAR); - tokens.push_back(line.substr(start, end - start)); - start = end + 1; - } - - /* create new file and add it to last directory*/ - if (ch == 'f') - { - if (tokens.size() != 6) - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " File token count wrong: " << tokens.size(); - std::cerr << std::endl; - for(unsigned int i = 0; i < tokens.size(); ++i) - { - std::cerr << "\tToken[" << i << "]:" << tokens[i]; - std::cerr << std::endl; - } - -#endif - goto error; - } - nfile = new FileEntry(); - registerEntry((void*)nfile) ; - nfile->name = tokens[0]; - nfile->hash = RsFileHash(tokens[1]); - nfile->size = atoll(tokens[2].c_str()); - nfile->modtime = atoi(tokens[3].c_str()); - nfile->pop = atoi(tokens[4].c_str()); - nfile->updtime = atoi(tokens[5].c_str()); - nfile->parent = ndir; - if (ndir) { - nfile->row = ndir->subdirs.size() + ndir->files.size(); - ndir->files[nfile->name] = nfile; - } - - } - /* create new dir and add to stack */ - else if (ch == 'd') - { - if (tokens.size() != 6) - { -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << " Dir token count wrong: " << tokens.size(); - std::cerr << std::endl; -#endif - goto error; - } - ndir = new DirEntry(); - registerEntry((void*)ndir) ; - ndir->name = tokens[0]; - ndir->path = tokens[1]; - ndir->size = atoi(tokens[2].c_str()); - ndir->modtime = atoi(tokens[3].c_str()); - ndir->pop = atoi(tokens[4].c_str()); - ndir->updtime = atoi(tokens[5].c_str()); - if (!dirlist.empty()) - { - ndir->parent = (dirlist.back()); - ndir->row = dirlist.back()->subdirs.size(); - dirlist.back()->subdirs[ndir->name] = ndir; - } - dirlist.push_back(ndir); - } - } - } - - updateHashIndex() ; - - return 1; - - /* parse error encountered */ -error: -#ifdef FI_DEBUG - std::cerr << "loadIndex error parsing saved file: " << filename; - std::cerr << std::endl; -#endif - return 0; -} - - -int FileIndex::saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size,const std::set& forbidden_dirs) -{ - std::string filenametmp = filename + ".tmp" ; - std::string s; - - size = 0 ; - - /* print version and header */ - s += "# FileIndex version 0.1\n"; - s += "# Dir: d name, path, parent, size, modtime, pop, updtime;\n"; - s += "# File: f name, hash, size, modtime, pop, updtime;\n"; - s += "#\n"; - - /* begin recusion */ - root->writeDirInfo(s) ; - - std::map::iterator it; - for(it = root->subdirs.begin(); it != root->subdirs.end(); ++it) - { -#ifdef FI_DEBUG - std::cout << "writting root directory: name=" << it->second->name << ", path=" << it->second->path << std::endl ; -#endif - if(forbidden_dirs.find(it->second->name) != forbidden_dirs.end()) - { -#ifdef FI_DEBUG - std::cerr << " will be suppressed." << std::endl ; -#endif - } - else - { -#ifdef FI_DEBUG - std::cerr << " will be saved." << std::endl ; -#endif - (it->second)->saveEntry(s); - } - } - - root->writeFileInfo(s) ; // this should never do anything - - /* signal to pop directory from stack in loadIndex() */ - s += "-\n"; - - // now compress the data. -#ifdef FI_DEBUG - std::cerr << "FileIndex::saveIndex(): compressign data." << std::endl; -#endif - - uint8_t *compressed_data = NULL ; - uint32_t compressed_data_size = 0 ; - - if(!RsCompress::compress_memory_chunk((unsigned char *)s.c_str(),s.length(),compressed_data,compressed_data_size)) - { - std::cerr << "(EE) ERROR in file list compression ! file list can't be saved" << std::endl; - free(compressed_data); - return false ; - } - - fileHash = RsDirUtil::sha1sum((unsigned char *)compressed_data,compressed_data_size); - -#ifdef FI_DEBUG - std::cerr << " file = " << filename << std::endl; - std::cerr << " old size = " << s.length() << std::endl; - std::cerr << " new size = " << compressed_data_size << std::endl; - std::cerr << " hash = " << fileHash << std::endl; -#endif - -// /* calculate sha1 hash */ -// SHA_CTX *sha_ctx = new SHA_CTX; -// SHA1_Init(sha_ctx); -// SHA1_Update(sha_ctx, s.c_str(), s.length()); -// SHA1_Final(&sha_buf[0], sha_ctx); -// delete sha_ctx; -// -// for(int i = 0; i < SHA_DIGEST_LENGTH; ++i) -// { -// rs_sprintf_append(fileHash, "%02x", (unsigned int) (sha_buf[i])); -// } - - /* finally, save to file */ - - FILE *file = RsDirUtil::rs_fopen(filenametmp.c_str(), "wb"); - if (file == NULL) - { - std::cerr << "FileIndex::saveIndex error opening file for writting: " << filename << ". Giving up." << std::endl; - return 0; - } - uint32_t outwritten ; - - if(compressed_data_size != (outwritten=fwrite(compressed_data,1,compressed_data_size,file))) - { - std::cerr << "FileIndex::saveIndex error. File not entirely written. Only " << outwritten << " bytes wrote out of " << compressed_data_size << " check for disk full, or disk quotas." << std::endl; - fclose(file); - return 0; - } - - fclose(file); - free(compressed_data) ; - - // Use a temp file name so that the file is never half saved. - // - if(!RsDirUtil::renameFile(filenametmp,filename)) - return false ; - - /* get the size out */ - struct stat64 buf; - - if(-1 == stat64(filename.c_str(), &buf)) - { - std::cerr << "Can't determine size of file " << filename << ": errno = " << errno << std::endl ; - return false ; - } - - size=buf.st_size; - - return true; -} - - -std::string FixName(const std::string& _in) -{ - /* replace any , with _ */ - std::string in(_in) ; - for(unsigned int i = 0; i < in.length(); ++i) - { - if (in[i] == FILE_CACHE_SEPARATOR_CHAR) - { - in[i] = '_'; - } - } - return in; -} - -void DirEntry::writeDirInfo(std::string& s) -{ - /* print node info */ - rs_sprintf_append(s, "d%s%c%s%c%lld%c%ld%c%d%c%ld%c\n", - FixName(name).c_str(), FILE_CACHE_SEPARATOR_CHAR, - FixName(path).c_str(), FILE_CACHE_SEPARATOR_CHAR, - size, FILE_CACHE_SEPARATOR_CHAR, - modtime, FILE_CACHE_SEPARATOR_CHAR, - pop, FILE_CACHE_SEPARATOR_CHAR, - updtime, FILE_CACHE_SEPARATOR_CHAR); -} - -void DirEntry::writeFileInfo(std::string& s) -{ - /* print file info */ - std::map::iterator fit; - for(fit = files.begin(); fit != files.end(); ++fit) - { - rs_sprintf_append(s, "f%s%c%s%c%lld%c%ld%c%d%c%ld%c\n", - FixName((fit->second)->name).c_str(), FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->hash.toStdString().c_str(), FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->size, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->modtime, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->pop, FILE_CACHE_SEPARATOR_CHAR, - (fit->second)->updtime, FILE_CACHE_SEPARATOR_CHAR); - } -} - -/* recusive function for traversing the dir tree in preorder */ -int DirEntry::saveEntry(std::string &s) -{ - writeDirInfo(s) ; - - std::map::iterator it; - for(it = subdirs.begin(); it != subdirs.end(); ++it) - { - (it->second)->saveEntry(s); - } - - writeFileInfo(s) ; - - /* signal to pop directory from stack in loadIndex() */ - s += "-\n"; - return 1; -} - - -int FileIndex::searchHash(const RsFileHash& hash, std::list &results) const -{ -#ifdef FI_DEBUG - std::cerr << "FileIndex::searchHash(" << hash << ")"; - std::cerr << std::endl; -#endif - - std::map::const_iterator it = _file_hashes.find(hash) ; - - if(it!=_file_hashes.end() && isValid((void*)it->second)) - results.push_back(it->second) ; - -#ifdef OLD_CODE_PLZ_REMOVE - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - /* add subdirs to stack */ - std::map::iterator it; - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - std::map::iterator fit; - /* search in current dir */ - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - if (hash == (fit->second)->hash) - { - results.push_back(fit->second); -#ifdef FI_DEBUG - std::cerr << "FileIndex::searchHash(" << hash << ")"; - std::cerr << " found: " << fit->second->name; - std::cerr << std::endl; -#endif - } - } - } -#endif - - return 0; -} - - -int FileIndex::searchTerms(const std::list& terms, std::list &results) const -{ - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - /* iterators */ - std::map::iterator it; - std::map::iterator fit; - std::list::const_iterator iter; - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - for (iter = terms.begin(); iter != terms.end(); ++iter) { - std::string::const_iterator it2; - const std::string &str1 = ndir->name; - const std::string &str2 = *iter; - it2 = std::search(str1.begin(), str1.end(), str2.begin(), str2.end(), CompareCharIC()); - if (it2 != str1.end()) { - /* first search to see if its parent is in the results list */ - bool addDir = true; - for (std::list::iterator rit(results.begin()); rit != results.end() && addDir; ++rit) { - DirEntry *de = dynamic_cast(*rit); - if (de && (de == root)) - continue; - if (de && (de == ndir->parent)) - addDir = false; - } - if (addDir) { - results.push_back((FileEntry *) ndir); - break; - } - } - } - - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - /* cycle through terms */ - for(iter = terms.begin(); iter != terms.end(); ++iter) - { - /* always ignore case */ - std::string::const_iterator it2 ; - const std::string &str1 = fit->second->name; - const std::string &str2 = (*iter); - - it2 = std::search( str1.begin(), str1.end(), - str2.begin(), str2.end(), CompareCharIC() ); - if (it2 != str1.end()) - { - results.push_back(fit->second); - break; - } - /* original case specific term search ****** - if (fit->second->name.find(*iter) != std::string::npos) - { - results.push_back(fit->second); - break; - } - ************/ - } - } - } //while - - return 0; -} - -int FileIndex::searchBoolExp(Expression * exp, std::list &results) const -{ - DirEntry *ndir = NULL; - std::list dirlist; - dirlist.push_back(root); - - /* iterators */ - std::map::iterator it; - std::map::iterator fit; - std::list::const_iterator iter; - - while(!dirlist.empty()) - { - ndir = dirlist.back(); - dirlist.pop_back(); - for(it = ndir->subdirs.begin(); it != ndir->subdirs.end(); ++it) - { - dirlist.push_back(it->second); - } - - for(fit = ndir->files.begin(); fit != ndir->files.end(); ++fit) - { - /*Evaluate the boolean expression and add it to the results if its true*/ - bool ret = exp->eval(fit->second); - if (ret == true){ - results.push_back(fit->second); - } - } - } //while - - return 0; -} - -uint32_t FileIndex::getType(void *ref) -{ - if(ref == NULL) - return DIR_TYPE_ROOT ; - - if(!isValid(ref)) - return DIR_TYPE_ROOT ; - - return static_cast(ref)->type() ; -} - -bool FileIndex::extractData(const std::string& fpath,DirDetails& details) const -{ - void *ref = findRef(fpath) ; - - if(ref == NULL) - return false ; - - return extractData(ref,details) ; -} - -void *FileIndex::findRef(const std::string& fpath) const -{ - DirEntry *parent = root->findDirectory(fpath); - - std::cerr << "findRef() Called on " << fpath << std::endl; - - if (!parent) - { -//#ifdef FI_DEBUG - std::cerr << "FileIndex::updateFileEntry() NULL parent"; - std::cerr << std::endl; -//#endif - return NULL; - } - std::cerr << "Found parent directory: " << std::endl; - std::cerr << " parent.name = " << parent->name << std::endl; - std::cerr << " parent.path = " << parent->path << std::endl; - - if(parent->path == fpath) // directory! - { - std::cerr << " fpath is a directory. Returning parent!" << std::endl; - return parent ; - } - else - { - std::cerr << " fpath is a file. Looking into parent directory..." << std::endl; - /* search in current dir */ - for(std::map::iterator fit = parent->files.begin(); fit != parent->files.end(); ++fit) - { - std::cerr << " trying " << parent->path + "/" + fit->second->name << std::endl; - if(parent->path + "/" + fit->second->name == fpath) - { - std::cerr << " found !" << std::endl; - return fit->second ; - } - } - - std::cerr << " (EE) not found !" << std::endl; - return NULL ; - } -} - -bool FileIndex::extractData(void *ref,DirDetails& details) -{ - if(!isValid(ref)) - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() asked for an invalid pointer " << (void*)ref << std::endl; -#endif - return false ; - } - - FileEntry *file = static_cast(ref); - DirEntry *dir = (file->hash.isNull())?static_cast(file):NULL ; // This is a hack to avoid doing a dynamic_cast - - details.children.clear() ; - time_t now = time(NULL) ; - - if (dir!=NULL) /* has children --- fill */ - { -#ifdef FI_DEBUG - std::cerr << "FileIndex::extractData() ref=dir" << std::endl; -#endif - /* extract all the entries */ - for(std::map::const_iterator dit(dir->subdirs.begin()); dit != dir->subdirs.end(); ++dit) - { - DirEntry *dirEntry = dit->second; - - DirStub stub; - stub.type = DIR_TYPE_DIR; - stub.name = dirEntry -> name; - stub.ref = dirEntry; - - details.children.push_back(stub); - } - - for(std::map::const_iterator fit(dir->files.begin()); fit != dir->files.end(); ++fit) - { - FileEntry *fileEntry = fit->second; - - DirStub stub; - stub.type = DIR_TYPE_FILE; - stub.name = fileEntry -> name; - stub.ref = fileEntry; - - details.children.push_back(stub); - } - - if(dir->parent == NULL) - details.type = DIR_TYPE_PERSON ; - else - details.type = DIR_TYPE_DIR; - details.hash.clear() ; - details.count = dir->subdirs.size() + dir->files.size(); - details.min_age = now - dir->most_recent_time ; - } - else - { -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() ref=file" << std::endl; -#endif - details.type = DIR_TYPE_FILE; - details.count = file->size; - details.min_age = now - file->modtime ; - } - -#ifdef FI_DEBUG - std::cerr << "FileIndexStore::extractData() name: " << file->name << std::endl; -#endif - details.ref = file; - details.hash = file->hash; - details.age = now - file->modtime; - details.flags.clear() ; - - /* find parent pointer, and row */ - details.parent = file->parent ; - - details.prow = (file->parent==NULL)?0:file->parent->row ; - - if(details.type == DIR_TYPE_DIR) - { - details.name = file->name; - details.path = dir->path; - } - else - { - details.name = file->name; - details.path = (file->parent==NULL)?"":file->parent->path; - } - - /* find peer id */ - FileEntry *f ; - for(f=file;f->parent!=NULL;f=f->parent) ; - - details.id = static_cast(f)->id; // The topmost parent is necessarily a personEntrY, so we can avoid a dynamic_cast. - -#ifdef FI_DEBUG - assert(details.parent != details.ref) ; - std::cout << "details: ref=" << (void*)ref << ", prow=" << details.prow << ", parent=" << (void*)details.parent << ", children=" ; - for(std::list::iterator it(details.children.begin());it!=details.children.end();++it) - std::cout << " " << (void*)it->ref ; - std::cout << std::endl ; -#endif - - return true; -} - diff --git a/libretroshare/src/dbase/findex.h b/libretroshare/src/dbase/findex.h deleted file mode 100644 index 87ad257d9..000000000 --- a/libretroshare/src/dbase/findex.h +++ /dev/null @@ -1,278 +0,0 @@ -/* - * RetroShare FileCache Module: findex.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef FILE_INDEX_H -#define FILE_INDEX_H - -#include -#include -#include -#if __MACH__ -#include -#else -#include -#endif -#include -#include -#include -#include "retroshare/rstypes.h" - -/****************************************************************************************** - * The Key Data Types for the File Index: - - FileEntry : Information about a single file. - DirEntry : Information about a directory and its children - PersonEntry : Information the root of a FileIndex. - - FileIndex : A Collection of root directories, with a set of functions to manipulate them. - In terms of retroshare, There will be a single 'Local' FileIndex used to store - the shared files, and a set of 'Remote' FileIndices which are used to store - the available files of all the peers. - -******************************************************************************************/ -/****************************************************************************************** - STILL TODO: - - (1) Load/Store a FileIndex to file... - int FileIndex::loadIndex(FILE *input); - int FileIndex::saveIndex(FILE *input); - - This can be done in a recursive manner, or handled completely within FileIndex. - - (2) Search Functions for Partial File Names and Hashes. - - int FileIndex::searchHash(std::string hash, std::list &results); - int FileIndex::searchTerms(std::list terms, std::list &results); - - This is probably best done in a recursive manner. - - The search could also be extended to handle complex Boolean searches such as : - match (size > 100K) && (name contains 'Blue') .... if anyone is interested. - But this can get quite complicated, and can be left to a later date. - -******************************************************************************************/ - - -/****************************************************************************************** - * FileEntry - *****************************************************************************************/ - -#include - -class DirEntry; - -class FileEntry: public RsMemoryManagement::SmallObject -{ - public: - FileEntry() - : size(0), modtime(0), pop(0), updtime(0), parent(NULL), row(0) - { return; } - - virtual ~FileEntry() { return; } - virtual uint32_t type() const { return DIR_TYPE_FILE ; } - -virtual int print(std::string &out); - - /* Data */ - std::string name; - RsFileHash hash; - uint64_t size; /* file size */ - time_t modtime; /* modification time - most recent mod time for a sub entry for dirs */ - int pop; /* popularity rating */ - - time_t updtime; /* last updated */ - - /* References for easy manipulation */ - DirEntry *parent; - int row; - std::list parent_groups ; -}; - -/****************************************************************************************** - * DirEntry - *****************************************************************************************/ - -class DirEntry: public FileEntry -{ - public: - - DirEntry() : most_recent_time(0) {} - /* cleanup */ -virtual ~DirEntry(); - - /* update local entries */ -DirEntry * updateDir(const FileEntry& fe, time_t updtime); -FileEntry * updateFile(const FileEntry& fe, time_t updtime); - - - virtual uint32_t type() const { return DIR_TYPE_DIR ; } -int checkParentPointers(); -int updateChildRows(); - - /* remove local entries */ -int removeFile(const std::string& name); -int removeDir(const std::string& name); -int removeOldDir(const std::string& name, time_t old); /* checks ts first */ - - /* recursive cleanup */ -int removeOldEntries(time_t old, bool recursive); - - /* recursive searches */ -DirEntry * findOldDirectory(time_t old); -DirEntry * findDirectory(const std::string& path); - - /* recursive update directory mod/pop values */ -int updateDirectories(const std::string& path, int pop, int modtime); - - /* output */ -int print(std::string &out); - -int saveEntry(std::string &out); -void writeDirInfo(std::string&); -void writeFileInfo(std::string&); - - /* Data */ - std::string path; /* full path (includes name) */ - std::map subdirs; - std::map files; - - time_t most_recent_time; /* last updated */ - - /* Inherited members from FileEntry: - int size - count for dirs - std::string name; - directory name - std::string hash; - not used - int size; - not used - int modtime; - most recent modication time of any child file (recursive) - int pop; - most popular child file (recursive) - int updtime; - last updated - */ - -}; - -/****************************************************************************************** - * PersonEntry - *****************************************************************************************/ - -class PersonEntry: public DirEntry -{ - public: - /* cleanup */ - PersonEntry(const RsPeerId& pid) : id(pid) { return; } -virtual ~PersonEntry() { return; } - -DirEntry &operator=(DirEntry &src) -{ - DirEntry *pdest = this; - (*pdest) = src; - return *this; -} - virtual uint32_t type() const { return DIR_TYPE_PERSON ; } - - /* Data */ - RsPeerId id; - - /* Inherited members from FileEntry: - int size - count for dirs - std::string name; - directory name - std::string hash; - not used - int size; - not used - int modtime; - most recent modication time of any child file (recursive) - int pop; - most popular child file (recursive) - int updtime; - last updated - */ - -}; - -/****************************************************************************************** - * FileIndex - *****************************************************************************************/ - -class Expression; - -class FileIndex -{ - public: - FileIndex(const RsPeerId& pid); - ~FileIndex(); - - /* control root entries */ - int setRootDirectories(const std::list &inlist, time_t utime); - int getRootDirectories(std::list &outlist); - - /* update (index building) */ - DirEntry * updateDirEntry(const std::string& path, const FileEntry& fe, time_t utime); - FileEntry * updateFileEntry(const std::string& path, const FileEntry& fe, time_t utime); - - DirEntry * findOldDirectory(time_t old); /* finds directories older than old */ - int removeOldDirectory(const std::string& fpath, const std::string& name, time_t old); - - int cleanOldEntries(time_t old); /* removes entries older than old */ - - /* debug */ - int printFileIndex(std::string &out); - int printFileIndex(std::ostream &out); - - /* load/save to file */ - int loadIndex(const std::string& filename, const RsFileHash &expectedHash, uint64_t size); - int saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size, const std::set& forbidden_roots); - - /* search through this index */ - int searchTerms(const std::list& terms, std::list &results) const; - int searchHash(const RsFileHash& hash, std::list &results) const; - int searchBoolExp(Expression * exp, std::list &results) const; - - /// Recursively compute the maximum modification time of children. - /// Used to easily retrieve mose recent files. - // - void updateMaxModTime() ; - void RecursUpdateMaxModTime(DirEntry *) ; - - PersonEntry *root; - -#ifdef __MACH__ - static std::unordered_set _pointers ; -#else - static std::tr1::unordered_set _pointers ; -#endif - static void registerEntry(void*p) ; - static void unregisterEntry(void*p) ; - static bool isValid(void*p) ; - - /// Fills up details from the data contained in ref. - // - static bool extractData(void *ref,DirDetails& details) ; - static uint32_t getType(void *ref) ; - - void *findRef(const std::string& path) const ; - bool extractData(const std::string& path,DirDetails& details) const ; - - void updateHashIndex() ; - void recursUpdateHashIndex(DirEntry *) ; - - std::map _file_hashes ; -}; - - -#endif - diff --git a/libretroshare/src/dbase/fistore.cc b/libretroshare/src/dbase/fistore.cc deleted file mode 100644 index 00d70a214..000000000 --- a/libretroshare/src/dbase/fistore.cc +++ /dev/null @@ -1,459 +0,0 @@ -/* - * RetroShare FileCache Module: fistore.cc - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include - -#include "rsserver/p3face.h" -#include "dbase/fistore.h" -#include "retroshare/rsexpr.h" -#include "retroshare/rsfiles.h" -#include "serialiser/rsserviceids.h" -#include "pqi/p3peermgr.h" -#include "pqi/p3notify.h" - -FileIndexStore::FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, - p3PeerMgr *cnmgr, RsPeerId ownid, std::string cachedir) - :CacheStore(RS_SERVICE_TYPE_FILE_INDEX, false, cs, cft, cachedir), - localId(ownid), localindex(NULL), mPeerMgr(cnmgr) -{ - return; -} - -FileIndexStore::~FileIndexStore() -{ - /* clean up the Index */ - return; -} - -/*** - * #define FIS_DEBUG2 1 - * #define FIS_DEBUG 1 - **/ - - /* actual load, once data available */ -int FileIndexStore::loadCache(const RsCacheData &data) -{ - -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() hash: " << data.hash << std::endl; - std::cerr << "FileIndexStore::loadCache() path: " << data.path << std::endl; - std::cerr << "FileIndexStore::loadCache() name: " << data.name << std::endl; - std::cerr << "FileIndexStore::loadCache() size: " << data.size << std::endl; -#endif - - /* do Callback */ - AboutToModify(); - - /* lock it up */ - lockData(); - - FileIndex *fiold = NULL; - bool local = (data.pid == localId); - - std::map::iterator it; - /* remove old cache */ - if (local) - { - fiold = localindex; - localindex = NULL; - } - else if (indices.end() != (it = indices.find(data.pid))) - { - fiold = it->second; - indices.erase(it); - //delete fi; - } - - if(mPeerMgr->isFriend(data.pid)) - { - // We discard file lists from non friends. This is the place to remove file lists of deleted friends - // from the cache. Doing this, the file list still shows in a session where we deleted a friend, but will be removed - // at next restart. - // - - /* load Cache */ - FileIndex *finew = new FileIndex(data.pid); - - if (finew->loadIndex(data.path + '/' + data.name, data.hash, data.size)) - { -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() Succeeded!" << std::endl; -#endif - /* This is not the place to set the peername. - * It is a hack, which makes it programmatically impossible - * to get the file data out.... - * - * peername should not be used in dbase. - */ - finew->root->name = data.pid.toStdString(); - - if (local) - { - localindex = finew; - } - else - { - indices[data.pid] = finew; - } - delete fiold; - - /* store in tale */ - locked_storeCacheEntry(data); - } - else - { -#ifdef FIS_DEBUG2 - std::cerr << "FileIndexStore::loadCache() Failed!" << std::endl; -#endif - /* reinstall the old one! */ - delete finew; - if (fiold) - { - if (local) - { - localindex = fiold; - } - else - { - indices[data.pid] = fiold; - } - } - } - } -#ifdef FIS_DEBUG - else - std::cerr << "Discarding file list from deleted peer " << data.pid << std::endl ; -#endif - - /* need to correct indices(row) for the roots of the FileIndex */ - int i = 0; - for(it = indices.begin(); it != indices.end(); ++it) - { - (it->second)->root->row = i++; - it->second->FileIndex::updateMaxModTime() ; - } - if (localindex) - { - localindex->root->row = 0; - localindex->updateMaxModTime() ; - } - - unlockData(); - - ModCompleted(); - bool ret = false; - return ret; -} - - - /* Search Interface - For Directory Access */ -int FileIndexStore::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails& details) const -{ - lockData(); - - std::map::const_iterator it = indices.find(uid); - bool found = true; - - if (it != indices.end()) - found = it->second->extractData(path,details) ; - - unlockData(); - - return found ; - -#ifdef OLD_STUFF_TO_REMOVE - /* lock it up */ - lockData(); - - std::map::const_iterator it; - it = indices.find(uid); - bool found = true; - - if (it != indices.end()) - { - //DirEntry *fdir = (it->second).lookupDirectory(path); - /* translate it - */ - bool b = FileIndex::extractData((it->second)->root,details) ; - - found = found && b ; - } - else - found = false; - - unlockData(); - return found; -#endif -} - -int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const -{ - /* remove unused parameter warnings */ - (void) flags; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl; -#endif - - std::map::const_iterator pit; - - lockData(); - -// checked by FileIndex::extractData -// if(ref != NULL && !FileIndex::isValid(ref)) -// { -// unlockData() ; -// return false ; -// } - - /* so cast *ref to a DirEntry */ - /* root case */ - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() CHKS" << std::endl; - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - (pit->second)->root->checkParentPointers(); - } -#endif - - if (ref == NULL) - { -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::RequestDirDetails() ref=NULL (root)" << std::endl; -#endif - - /* get remote root entries */ - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - /* - */ - FileIndex *fileIndex = pit->second; - - DirStub stub; - stub.type = DIR_TYPE_PERSON; - stub.name = fileIndex->root->name; - stub.ref = fileIndex->root; - - details.children.push_back(stub); - } - details.parent = NULL; - details.prow = -1; - details.ref = NULL; - details.type = DIR_TYPE_ROOT; - details.name = ""; - details.hash.clear() ; - details.path = ""; - details.count = indices.size(); - details.age = 0; - details.flags.clear() ; - details.min_age = 0; - - unlockData(); - return true ; - } - - bool b = FileIndex::extractData(ref,details) ; - - unlockData(); - return b; -} -uint32_t FileIndexStore::getType(void *ref) const -{ - lockData() ; - uint32_t b = FileIndex::getType(ref) ; - unlockData(); - - return b; -} - -int FileIndexStore::SearchHash(const RsFileHash& hash, std::list &results) const -{ - lockData(); - results.clear() ; - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - - time_t now = time(NULL); - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash()" << std::endl; -#endif - for(pit = indices.begin(); pit != indices.end(); ++pit) - { -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash() Searching: Peer "; - std::cerr << pit->first << std::endl; -#endif - firesults.clear(); - - (pit->second)->searchHash(hash, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - FileDetail fd; - fd.id = pit->first; - fd.name = (*rit)->name; - fd.hash = (*rit)->hash; - fd.path = ""; /* TODO */ - fd.size = (*rit)->size; - fd.age = now - (*rit)->modtime; - fd.rank = (*rit)->pop; - - results.push_back(fd); - } - - } - - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchHash() Found " << results.size(); - std::cerr << " Results from " << indices.size() << " Peers" << std::endl; -#endif - - unlockData(); - return results.size(); -} - - -int FileIndexStore::SearchKeywords(std::list keywords, std::list &results,FileSearchFlags flags) const -{ - lockData(); - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - - results.clear() ; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::SearchKeywords()" << std::endl; -#endif - if(flags & RS_FILE_HINTS_REMOTE) - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - firesults.clear(); - - (pit->second)->searchTerms(keywords, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - - if(!FileIndex::extractData(*rit, dd)) - continue ; - - results.push_back(dd); - } - } - - if(flags & RS_FILE_HINTS_LOCAL) - if (localindex) - { - firesults.clear(); - - localindex->searchTerms(keywords, firesults); - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - - if(!FileIndex::extractData(*rit, dd)) - continue ; - - dd.id.clear() ; - results.push_back(dd); - } - - } - - unlockData(); - return results.size(); -} - - -int FileIndexStore::searchBoolExp(Expression * exp, std::list &results) const -{ - lockData(); - std::map::const_iterator pit; - std::list::iterator rit; - std::list firesults; - -#ifdef FIS_DEBUG - std::cerr << "FileIndexStore::searchBoolExp()" << std::endl; -#endif - for(pit = indices.begin(); pit != indices.end(); ++pit) - { - firesults.clear(); - - (pit->second)->searchBoolExp(exp, firesults); - - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - FileIndex::extractData(*rit, dd); - results.push_back(dd); - } - - } - - /* finally search local files */ - if (localindex) - { - firesults.clear(); - - localindex->searchBoolExp(exp, firesults); - - /* translate results */ - for(rit = firesults.begin(); rit != firesults.end(); ++rit) - { - DirDetails dd; - FileIndex::extractData(*rit, dd); - dd.id.clear() ; - results.push_back(dd); - } - - } - - - unlockData(); - return results.size(); -} - -int FileIndexStore::AboutToModify() -{ - RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - - return 1; -} - - -int FileIndexStore::ModCompleted() -{ - RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0); - - return 1; -} - - diff --git a/libretroshare/src/dbase/fistore.h b/libretroshare/src/dbase/fistore.h deleted file mode 100644 index e175adec8..000000000 --- a/libretroshare/src/dbase/fistore.h +++ /dev/null @@ -1,102 +0,0 @@ -/* - * RetroShare FileCache Module: fistore.h - * - * Copyright 2004-2007 by Robert Fernie. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#ifndef MRK_FILE_INDEX_STORE_H -#define MRK_FILE_INDEX_STORE_H - - -/********** - * Stores the FileCaches of the Peers - * must implement 'loadCache' to - * - * This class is also accessed by the GUI.... - * and the FileTransfer class. - * - */ - -class p3PeerMgr ; - -#include "dbase/findex.h" -#include "dbase/cachestrapper.h" -#include "retroshare/rsiface.h" - -class FileStoreResult -{ - public: - std::string id; - std::string path; - std::string hash; - std::string name; -}; - -class NotifyCallback -{ - public: - NotifyCallback() { return; } -virtual ~NotifyCallback() { return; } -virtual void AboutToModify() { return; } -virtual void ModCompleted() { return; } -}; - - -class Expression; - -class FileIndexStore: public CacheStore -{ - public: - - FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, p3PeerMgr *pmgr, RsPeerId ownid, std::string cachedir); -virtual ~FileIndexStore(); - - /* virtual functions overloaded by cache implementor */ -virtual int loadCache(const RsCacheData &data); /* actual load, once data available */ - - /* Search Interface - For FileTransfer Lookup */ - int SearchHash(const RsFileHash &hash, std::list &results) const; - - /* Search Interface - For Search Interface */ - int SearchKeywords(std::list terms, std::list &results,FileSearchFlags flags) const; - - /* Search Interface - for Adv Search Interface */ - int searchBoolExp(Expression * exp, std::list &results) const; - - - /* Search Interface - For Directory Access */ - int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) const; - int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const; - uint32_t getType(void *ref) const ; - - private: - int AboutToModify(); - int ModCompleted(); - - std::map indices; - - RsPeerId localId; - FileIndex *localindex; - - p3PeerMgr *mPeerMgr ; -}; - - -#endif