diff --git a/libretroshare/src/Makefile b/libretroshare/src/Makefile index 7ea5a6617..a34b64e5f 100644 --- a/libretroshare/src/Makefile +++ b/libretroshare/src/Makefile @@ -14,9 +14,10 @@ librs: make -C services librs make -C dht librs make -C upnp librs - make -C server librs + make -C ft librs make -C rsserver librs +# make -C server librs # make -C rsiface librs tests: @@ -28,10 +29,12 @@ tests: make -C services tests make -C dht tests make -C upnp tests - make -C server tests + make -C ft tests make -C rsserver tests make -C rsiface tests +# make -C server tests + clean: make -C util clean make -C tcponudp clean @@ -41,10 +44,12 @@ clean: make -C services clean make -C dht clean make -C upnp clean - make -C server clean + make -C ft clean make -C rsserver clean make -C rsiface clean +# make -C server clean + clobber: make -C util clobber make -C tcponudp clobber @@ -55,7 +60,9 @@ clobber: make -C dht clobber make -C upnp clobber make -C server clobber + make -C ft clobber make -C rsserver clobber make -C rsiface clobber -$(RM) lib/libretroshare.a +# make -C server clobber diff --git a/libretroshare/src/dbase/cachestrapper.cc b/libretroshare/src/dbase/cachestrapper.cc index 3889b4432..ee740a4f0 100644 --- a/libretroshare/src/dbase/cachestrapper.cc +++ b/libretroshare/src/dbase/cachestrapper.cc @@ -34,7 +34,6 @@ * #define CS_DEBUG 1 ***/ -#define CS_DEBUG 1 bool operator<(const CacheId &a, const CacheId &b) { diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index 6d0e9de75..c8f3bbf62 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -39,6 +39,7 @@ /*********** * #define FIM_DEBUG 1 ***********/ +#define FIM_DEBUG 1 FileIndexMonitor::FileIndexMonitor(CacheStrapper *cs, std::string cachedir, std::string pid) :CacheSource(RS_SERVICE_TYPE_FILE_INDEX, false, cs, cachedir), fi(pid), @@ -596,6 +597,17 @@ void FileIndexMonitor::setSharedDirectories(std::list dirs) { fiMutex.lock(); { /* LOCKED DIRS */ +#ifdef FIM_DEBUG + std::list::iterator it; + std::cerr << "FileIndexMonitor::setSharedDirectories() :\n"; + for(it = dirs.begin(); it != dirs.end(); it++) + { + std::cerr << "\t" << *it; + std::cerr << std::endl; + } +#endif + + pendingDirs = true; pendingDirList = dirs; diff --git a/libretroshare/src/dbase/findex.cc b/libretroshare/src/dbase/findex.cc index 912e269cc..d7b40e216 100644 --- a/libretroshare/src/dbase/findex.cc +++ b/libretroshare/src/dbase/findex.cc @@ -37,6 +37,9 @@ * #define FI_DEBUG 1 ****/ +#define FI_DEBUG 1 +#define FI_DEBUG_ALL 1 + DirEntry::~DirEntry() { @@ -609,7 +612,7 @@ int FileIndex::removeOldDirectory(std::string fpath, std::string name, time_t { /* path is to parent */ #ifdef FI_DEBUG_ALL - std::cerr << "FileIndex::removeOldDir() Path: \""; + std::cerr << "FileIndex::removeOldDirectory() Path: \""; std::cerr << fpath << "\"" << " + \"" << name << "\""; std::cerr << std::endl; #endif @@ -617,11 +620,21 @@ int FileIndex::removeOldDirectory(std::string fpath, std::string name, time_t /* because of this find - we cannot get a child of * root (which is what we want!) */ + DirEntry *parent = root->findDirectory(fpath); + /* for root directory case ... no subdir. */ + if (fpath == "") + { +#ifdef FI_DEBUG + std::cerr << "FileIndex::removeOldDirectory() removing a root dir"; + std::cerr << std::endl; +#endif + parent = root; + } if (!parent) { #ifdef FI_DEBUG - std::cerr << "FileIndex::removeOldDir() NULL parent"; + std::cerr << "FileIndex::removeOldDirectory() NULL parent"; std::cerr << std::endl; #endif return 0; diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index e7d76807a..5fe80b607 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -96,8 +96,10 @@ void ftController::run() sleep(1); #endif - std::cerr << "ftController::run()"; - std::cerr << std::endl; +#ifdef CONTROL_DEBUG + //std::cerr << "ftController::run()"; + //std::cerr << std::endl; +#endif /* tick the transferModules */ std::list done; @@ -254,6 +256,7 @@ bool ftController::FileRequest(std::string fname, std::string hash, /* check if we have the file */ FileInfo info; std::list::iterator it; + std::list::iterator pit; #ifdef CONTROL_DEBUG std::cerr << "ftController::FileRequest(" << fname << ","; @@ -313,17 +316,17 @@ bool ftController::FileRequest(std::string fname, std::string hash, #endif /* if the sources don't exist already - add in */ - for(it = info.peerIds.begin(); it != info.peerIds.end(); it++) + for(pit = info.peers.begin(); pit != info.peers.end(); pit++) { - std::cerr << "\tSource: " << *it; + std::cerr << "\tSource: " << pit->peerId; std::cerr << std::endl; if (srcIds.end() == std::find( - srcIds.begin(), srcIds.end(), *it)) + srcIds.begin(), srcIds.end(), pit->peerId)) { - srcIds.push_back(*it); + srcIds.push_back(pit->peerId); - std::cerr << "\tAdding in: " << *it; + std::cerr << "\tAdding in: " << pit->peerId; std::cerr << std::endl; } } @@ -563,6 +566,75 @@ bool ftController::FileDetails(std::string hash, FileInfo &info) } /* extract details */ + info.hash = hash; + info.fname = it->second.mName; + + /* get list of sources from transferModule */ + std::list peerIds; + std::list::iterator pit; + + it->second.mTransfer->getFileSources(peerIds); + + double totalRate; + uint32_t tfRate; + uint32_t state; + + bool isDownloading = false; + bool isSuspended = false; + + for(pit = peerIds.begin(); pit != peerIds.end(); pit++) + { + if (it->second.mTransfer->getPeerState(*pit, state, tfRate)) + { + TransferInfo ti; + switch(state) + { + case PQIPEER_INIT: + ti.status = FT_STATE_OKAY; + break; + case PQIPEER_NOT_ONLINE: + ti.status = FT_STATE_WAITING; + break; + case PQIPEER_DOWNLOADING: + isDownloading = true; + ti.status = FT_STATE_DOWNLOADING; + break; + case PQIPEER_IDLE: + ti.status = FT_STATE_OKAY; + break; + default: + case PQIPEER_SUSPEND: + isSuspended = true; + ti.status = FT_STATE_FAILED; + break; + } + + ti.tfRate = tfRate / 1024.0; + ti.peerId = *pit; + info.peers.push_back(ti); + totalRate += tfRate / 1024.0; + } + } + + if ((it->second).mCreator->finished()) + { + info.downloadStatus = FT_STATE_COMPLETE; + } + else if (isDownloading) + { + info.downloadStatus = FT_STATE_DOWNLOADING; + } + else if (isSuspended) + { + info.downloadStatus = FT_STATE_FAILED; + } + else + { + info.downloadStatus = FT_STATE_WAITING; + } + info.tfRate = totalRate; + info.size = (it->second).mSize; + info.transfered = (it->second).mCreator->getRecvd(); return true; diff --git a/libretroshare/src/ft/ftdbase.cc b/libretroshare/src/ft/ftdbase.cc index 8a8e7454b..f2a4bbd73 100644 --- a/libretroshare/src/ft/ftdbase.cc +++ b/libretroshare/src/ft/ftdbase.cc @@ -91,11 +91,21 @@ bool ftFiStore::search(std::string hash, uint64_t size, uint32_t hintflags, File info.fname = it->name; info.size = it->size; info.hash = it->hash; + } - info.peerIds.push_back(it->id); + + TransferInfo ti; + ti.peerId = it->id; + ti.name = it->name; + ti.tfRate = 0; + info.peers.push_back(ti); } } + /**** DEPENDS ON SOURCES! + info.downloadStatus = FT_STATE_COMPLETE: + ****/ + /* if the first flag is cleared, we've definitely * had a full match!. */ diff --git a/libretroshare/src/ft/ftextralist.cc b/libretroshare/src/ft/ftextralist.cc index d57326708..c5f7765fa 100644 --- a/libretroshare/src/ft/ftextralist.cc +++ b/libretroshare/src/ft/ftextralist.cc @@ -44,8 +44,8 @@ void ftExtraList::run() while (1) { #ifdef DEBUG_ELIST - std::cerr << "ftExtraList::run() Iteration"; - std::cerr << std::endl; + //std::cerr << "ftExtraList::run() Iteration"; + //std::cerr << std::endl; #endif now = time(NULL); diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index ef5fc31e5..9ed1fa3d8 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -5,6 +5,8 @@ * #define FILE_DEBUG 1 ******/ +#define FILE_DEBUG 1 + #define CHUNK_MAX_AGE 30 @@ -42,12 +44,15 @@ hash, uint64_t recvd): ftFileProvider(path,size,hash) bool ftFileCreator::getFileData(uint64_t offset, uint32_t chunk_size, void *data) { - RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - if (offset + chunk_size > mStart) { + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + if (offset + chunk_size > mStart) + { /* don't have the data */ return false; - } + } + } + return ftFileProvider::getFileData(offset, chunk_size, data); } @@ -64,6 +69,7 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data std::cerr << offset; std::cerr << ", " << chunk_size; std::cerr << ", " << data << ")"; + std::cerr << " this: " << this; std::cerr << std::endl; #endif /* dodgey checking outside of mutex... @@ -123,7 +129,7 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data /* * Notify ftFileChunker about chunks received */ - notifyReceived(offset,chunk_size); + locked_notifyReceived(offset,chunk_size); /* * FIXME HANDLE COMPLETION HERE - Any better way? @@ -136,6 +142,7 @@ int ftFileCreator::initializeFileAttrs() { std::cerr << "ftFileCreator::initializeFileAttrs() Filename: "; std::cerr << file_name; + std::cerr << " this: " << this; std::cerr << std::endl; /* @@ -195,9 +202,15 @@ ftFileCreator::~ftFileCreator() } -int ftFileCreator::notifyReceived(uint64_t offset, uint32_t chunk_size) +int ftFileCreator::locked_notifyReceived(uint64_t offset, uint32_t chunk_size) { /* ALREADY LOCKED */ +#ifdef FILE_DEBUG + std::cerr << "ftFileCreator::locked_notifyReceived( " << offset; + std::cerr << ", " << chunk_size << " )"; + std::cerr << " this: " << this; + std::cerr << std::endl; +#endif /* find the chunk */ std::map::iterator it; @@ -205,6 +218,13 @@ int ftFileCreator::notifyReceived(uint64_t offset, uint32_t chunk_size) bool isFirst = false; if (it == mChunks.end()) { +#ifdef FILE_DEBUG + std::cerr << "ftFileCreator::locked_notifyReceived() "; + std::cerr << " Failed to match to existing chunk - ignoring"; + std::cerr << std::endl; + + locked_printChunkMap(); +#endif return 0; /* ignoring */ } else if (it == mChunks.begin()) @@ -246,14 +266,21 @@ int ftFileCreator::notifyReceived(uint64_t offset, uint32_t chunk_size) bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk) { RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - std::cerr << "ffc::getMissingChunk(...,"<< chunk << ")"<< std::endl; +#ifdef FILE_DEBUG + std::cerr << "ffc::getMissingChunk(...,"<< chunk << ")"; + std::cerr << " this: " << this; + std::cerr << std::endl; + locked_printChunkMap(); +#endif /* check start point */ if (mStart == mSize) { +#ifdef FILE_DEBUG std::cerr << "ffc::getMissingChunk() File Done"; std::cerr << std::endl; +#endif return false; } @@ -267,8 +294,10 @@ bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk) /* very simple algorithm */ if (it->second.ts < old) { +#ifdef FILE_DEBUG std::cerr << "ffc::getMissingChunk() ReAlloc"; std::cerr << std::endl; +#endif /* retry this one */ it->second.ts = ts; @@ -279,10 +308,12 @@ bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk) } } +#ifdef FILE_DEBUG std::cerr << "ffc::getMissingChunk() new Alloc"; std::cerr << " mStart: " << mStart << " mEnd: " << mEnd; std::cerr << "mSize: " << mSize; std::cerr << std::endl; +#endif /* else allocate a new chunk */ if (mSize - mEnd < chunk) @@ -293,12 +324,48 @@ bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk) if (chunk > 0) { +#ifdef FILE_DEBUG + std::cerr << "ffc::getMissingChunk() Allocated " << chunk; + std::cerr << " offset: " << offset; + std::cerr << std::endl; + std::cerr << " mStart: " << mStart << " mEnd: " << mEnd; + std::cerr << "mSize: " << mSize; + std::cerr << std::endl; +#endif + mChunks[offset] = ftChunk(offset, chunk, ts); } return true; /* cos more data to get */ } + +bool ftFileCreator::locked_printChunkMap() +{ +#ifdef FILE_DEBUG + std::cerr << "ftFileCreator::locked_printChunkMap()"; + std::cerr << " this: " << this; + std::cerr << std::endl; +#endif + + /* check start point */ + std::cerr << "Size: " << mSize << " Start: " << mStart << " End: " << mEnd; + std::cerr << std::endl; + std::cerr << "\tOutstanding Chunks (in the middle)"; + std::cerr << std::endl; + + std::map::iterator it; + time_t ts = time(NULL); + for(it = mChunks.begin(); it != mChunks.end(); it++) + { + std::cerr << "\tChunk [" << it->second.offset << "] size: "; + std::cerr << it->second.chunk; + std::cerr << " Age: " << ts - it->second.ts; + std::cerr << std::endl; + } + return true; +} + /*********************************************************** * * ftChunk methods diff --git a/libretroshare/src/ft/ftfilecreator.h b/libretroshare/src/ft/ftfilecreator.h index 454991ea0..0451c25b5 100644 --- a/libretroshare/src/ft/ftfilecreator.h +++ b/libretroshare/src/ft/ftfilecreator.h @@ -63,7 +63,8 @@ virtual int initializeFileAttrs(); private: - int notifyReceived(uint64_t offset, uint32_t chunk_size); + bool locked_printChunkMap(); + int locked_notifyReceived(uint64_t offset, uint32_t chunk_size); /* * structure to track missing chunks */ diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index e33818732..5866e61f4 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -452,7 +452,9 @@ bool ftServer::sendDataRequest(std::string peerId, std::string hash, return true; } -const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */ +//const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */ +//const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */ +const uint32_t MAX_FT_CHUNK = 8 * 1024; /* 16K */ /* Server Send */ bool ftServer::sendData(std::string peerId, std::string hash, uint64_t size, @@ -467,8 +469,8 @@ bool ftServer::sendData(std::string peerId, std::string hash, uint64_t size, #ifdef SERVER_DEBUG std::cerr << "ftServer::sendData() to " << peerId << std::endl; std::cerr << "hash: " << hash; - std::cerr << " offset: " << offset; - std::cerr << " chunk: " << chunk; + std::cerr << " offset: " << baseoffset; + std::cerr << " chunk: " << chunksize; std::cerr << " data: " << data; std::cerr << std::endl; #endif @@ -527,6 +529,10 @@ bool ftServer::sendData(std::string peerId, std::string hash, uint64_t size, } +/* NB: The rsCore lock must be activated before calling this. + * This Lock should be moved lower into the system... + * most likely destination is in ftServer. + */ int ftServer::tick() { rslog(RSL_DEBUG_BASIC, ftserverzone, @@ -586,7 +592,7 @@ bool ftServer::handleCacheData() int i_init = 0; #ifdef SERVER_DEBUG - std::cerr << "ftServer::handleCacheData()" << std::endl; + //std::cerr << "ftServer::handleCacheData()" << std::endl; #endif while((ci = mP3iface -> GetSearchResult()) != NULL) { diff --git a/libretroshare/src/ft/ftserver1test.cc b/libretroshare/src/ft/ftserver1test.cc index 126ad804d..406105e52 100644 --- a/libretroshare/src/ft/ftserver1test.cc +++ b/libretroshare/src/ft/ftserver1test.cc @@ -145,7 +145,7 @@ int main(int argc, char **argv) std::list::iterator fit; std::cerr << "Point 3" << std::endl; - P3Hub *testHub = new P3Hub(); + P3Hub *testHub = new P3Hub(0, NULL); testHub->start(); std::cerr << "Point 4" << std::endl; diff --git a/libretroshare/src/ft/ftserver2test.cc b/libretroshare/src/ft/ftserver2test.cc index 9920cede1..db58aab9a 100644 --- a/libretroshare/src/ft/ftserver2test.cc +++ b/libretroshare/src/ft/ftserver2test.cc @@ -165,7 +165,16 @@ int main(int argc, char **argv) std::list baseFriendList, friendList; std::list::iterator fit; - P3Hub *testHub = new P3Hub(); + + /* Add in serialiser */ + + RsSerialiser *rss = new RsSerialiser(); + rss->addSerialType(new RsFileItemSerialiser()); + rss->addSerialType(new RsCacheItemSerialiser()); + rss->addSerialType(new RsServiceSerialiser()); + + + P3Hub *testHub = new P3Hub(0, rss); testHub->start(); /* Setup Base Friend Info */ diff --git a/libretroshare/src/ft/ftserver3test.cc b/libretroshare/src/ft/ftserver3test.cc index c7226a82a..7f1948790 100644 --- a/libretroshare/src/ft/ftserver3test.cc +++ b/libretroshare/src/ft/ftserver3test.cc @@ -175,7 +175,14 @@ int main(int argc, char **argv) std::list baseFriendList, friendList; std::list::iterator fit; - P3Hub *testHub = new P3Hub(); + /* Add in Serialiser Test + */ + RsSerialiser *rss = new RsSerialiser(); + rss->addSerialType(new RsFileItemSerialiser()); + rss->addSerialType(new RsCacheItemSerialiser()); + rss->addSerialType(new RsServiceSerialiser()); + + P3Hub *testHub = new P3Hub(0, rss); testHub->start(); /* Setup Base Friend Info */ @@ -317,8 +324,8 @@ int main(int argc, char **argv) while(1) { - std::cerr << "ftserver2test::sleep()"; - std::cerr << std::endl; + //std::cerr << "ftserver3test::sleep()"; + //std::cerr << std::endl; sleep(1); /* tick the connmgrs */ @@ -372,7 +379,7 @@ void *do_server_test_thread(void *data) std::string oId = oServer->OwnId(); /* create Expression */ - uint64_t minFileSize = 10000; + uint64_t minFileSize = 100000; //SizeExpression se(Greater, minFileSize); SizeExpression se(Smaller, minFileSize); Expression *expr = &se; @@ -437,9 +444,9 @@ void *do_server_test_thread(void *data) } /* Give it a while to transfer */ - for(int i = 0; i < 10; i++) + for(int i = 0; i < 300; i++) { - std::cerr << "Waiting 10 seconds to transfer"; + std::cerr << "Waited " << i * 10 << " seconds for transfer"; std::cerr << std::endl; sleep(10); } diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index 5f4d05edd..b5fed097f 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -27,11 +27,15 @@ * #define FT_DEBUG 1 ******/ +#define FT_DEBUG 1 + #include "fttransfermodule.h" ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm, ftController *c) :mFileCreator(fc), mMultiplexor(dm), mFtController(c), mFlag(0) { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + mHash = mFileCreator->getHash(); mSize = mFileCreator->getFileSize(); mFileStatus.hash = mHash; @@ -45,8 +49,11 @@ ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm, ftCon ftTransferModule::~ftTransferModule() {} + bool ftTransferModule::setFileSources(std::list peerIds) { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + mFileSources.clear(); #ifdef FT_DEBUG @@ -73,8 +80,20 @@ bool ftTransferModule::setFileSources(std::list peerIds) return true; } +bool ftTransferModule::getFileSources(std::list &peerIds) +{ + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + std::map::iterator it; + for(it = mFileSources.begin(); it != mFileSources.end(); it++) + { + peerIds.push_back(it->first); + } + return true; +} + bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t maxRate) { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ #ifdef FT_DEBUG std::cerr << "ftTransferModule::setPeerState()"; std::cerr << " peerId: " << peerId; @@ -108,8 +127,30 @@ bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t m return true; } + +bool ftTransferModule::getPeerState(std::string peerId,uint32_t &state,uint32_t &tfRate) +{ + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + std::map::iterator mit; + mit = mFileSources.find(peerId); + + if (mit == mFileSources.end()) return false; + + state = (mit->second).state; + tfRate = (uint32_t) (mit->second).actualRate; + +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::getPeerState()"; + std::cerr << " peerId: " << peerId; + std::cerr << " state: " << state; + std::cerr << " tfRate: " << tfRate << std::endl; +#endif + return true; +} + uint32_t ftTransferModule::getDataRate(std::string peerId) { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ std::map::iterator mit; mit = mFileSources.find(peerId); if (mit == mFileSources.end()) @@ -139,6 +180,9 @@ bool ftTransferModule::recvFileData(std::string peerId, uint64_t offset, std::cerr << std::endl; #endif + { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + std::map::iterator mit; mit = mFileSources.find(peerId); @@ -176,7 +220,7 @@ bool ftTransferModule::recvFileData(std::string peerId, uint64_t offset, (mit->second).receivedSize += chunk_size; (mit->second).state = PQIPEER_IDLE; - + } /***** STACK MUTEX END ****/ return storeData(offset, chunk_size, data); } @@ -246,6 +290,10 @@ bool ftTransferModule::storeData(uint64_t offset, uint32_t chunk_size,void *data bool ftTransferModule::queryInactive() { + /* NB: Not sure about this lock... might cause deadlock. + */ + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + #ifdef FT_DEBUG std::cerr << "ftTransferModule::queryInactive()" << std::endl; #endif @@ -291,12 +339,20 @@ bool ftTransferModule::queryInactive() //file response received or peer side is just ready for download case PQIPEER_IDLE: pInfo->actualRate = pInfo->chunkSize/(ts-(pInfo->lastTS)); - if (pInfo->actualRate < pInfo->desiredRate/2) - { - req_size = pInfo->chunkSize * 2 ; + + if (pInfo->actualRate < pInfo->desiredRate) + { + if (pInfo->actualRate < pInfo->desiredRate/2) + { + req_size = pInfo->chunkSize * 2 ; + } + else + { + req_size = (uint32_t ) (pInfo->chunkSize * 1.1) ; + } } - else - { + else + { req_size = (uint32_t ) (pInfo->chunkSize * 0.9) ; } @@ -336,6 +392,8 @@ bool ftTransferModule::queryInactive() bool ftTransferModule::pauseTransfer() { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + /* std::map::iterator mit; for(mit = mOnlinePeers.begin(); mit != mOnlinePeers.end(); mit++) @@ -350,6 +408,7 @@ bool ftTransferModule::pauseTransfer() bool ftTransferModule::resumeTransfer() { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ /* std::map::iterator mit; for(mit = mOnlinePeers.begin(); mit != mOnlinePeers.end(); mit++) @@ -364,6 +423,7 @@ bool ftTransferModule::resumeTransfer() bool ftTransferModule::cancelTransfer() { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ mFileStatus.stat=ftFileStatus::PQIFILE_FAIL_CANCEL; return 1; @@ -388,7 +448,14 @@ int ftTransferModule::tick() #endif queryInactive(); - switch (mFlag) + + uint32_t flags = 0; + { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + flags = mFlag; + } + + switch (flags) { case 0: adjustSpeed(); @@ -409,12 +476,30 @@ int ftTransferModule::tick() void ftTransferModule::adjustSpeed() { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + std::map::iterator mit; + +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::adjustSpeed()"; + std::cerr << " Initial Desired Rate: " << desiredRate << " Actual Rate: " << actualRate; + std::cerr << std::endl; +#endif + + + for(mit = mFileSources.begin(); mit != mFileSources.end(); mit++) { if (((mit->second).state == PQIPEER_DOWNLOADING) || ((mit->second).state == PQIPEER_IDLE)) { + +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::adjustSpeed()"; + std::cerr << "\t" << mit->first << " Desired Rate: " << desiredRate << " Actual Rate: " << actualRate; + std::cerr << std::endl; +#endif + if ((actualRate < desiredRate) && ((mit->second).actualRate >= (mit->second).desiredRate)) { (mit->second).desiredRate *= 1.1; @@ -426,6 +511,11 @@ void ftTransferModule::adjustSpeed() } } } +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::adjustSpeed()"; + std::cerr << " Initial Desired Rate: " << desiredRate << " Actual Rate: " << actualRate; + std::cerr << std::endl; +#endif return; } diff --git a/libretroshare/src/ft/fttransfermodule.h b/libretroshare/src/ft/fttransfermodule.h index d69420951..621f0ccc7 100644 --- a/libretroshare/src/ft/fttransfermodule.h +++ b/libretroshare/src/ft/fttransfermodule.h @@ -132,6 +132,8 @@ public: //interface to download controller bool setFileSources(std::list peerIds); bool setPeerState(std::string peerId,uint32_t state,uint32_t maxRate); //state = ONLINE/OFFLINE + bool getFileSources(std::list &peerIds); + bool getPeerState(std::string peerId,uint32_t &state,uint32_t &tfRate); uint32_t getDataRate(std::string peerId); bool pauseTransfer(); bool resumeTransfer(); diff --git a/libretroshare/src/ft/pqitestor.cc b/libretroshare/src/ft/pqitestor.cc index 41697476d..f28489fd5 100644 --- a/libretroshare/src/ft/pqitestor.cc +++ b/libretroshare/src/ft/pqitestor.cc @@ -27,11 +27,18 @@ #include "pqi/p3connmgr.h" /****** - *#define HUB_DEBUG + *#define HUB_DEBUG 1 *****/ -P3Hub::P3Hub() +#define HUB_DEBUG 1 + +P3Hub::P3Hub(uint32_t flags, RsSerialiser *rss) + :mSerialiser(rss), mUseSerialiser(false) { + if (rss) + { + mUseSerialiser = true; + } return; } @@ -57,6 +64,54 @@ void P3Hub::addP3Pipe(std::string id, P3Pipe *pqi, p3ConnectMgr *mgr) } +RsItem *P3Hub::SerialiserPass(RsItem *inItem) +{ + /* pass through serialiser */ + + RsItem *item = NULL; + + uint32_t pktsize = mSerialiser->size(inItem); + void *ptr = malloc(pktsize); + +#ifdef HUB_DEBUG + std::cerr << "P3Hub::SerialiserPass() Expected Size: " << pktsize; + std::cerr << std::endl; +#endif + + if (!mSerialiser->serialise(inItem, ptr, &pktsize)) + { + +#ifdef HUB_DEBUG + std::cerr << "P3Hub::SerialiserPass() serialise Failed"; + std::cerr << std::endl; +#endif + + } + else + { + +#ifdef HUB_DEBUG + std::cerr << "P3Hub::SerialiserPass() serialise success, size: " << pktsize; + std::cerr << std::endl; +#endif + + item = mSerialiser->deserialise(ptr, &pktsize); + item->PeerId(inItem->PeerId()); + + if (!item) + { +#ifdef HUB_DEBUG + std::cerr << "P3Hub::SerialiserPass() deSerialise Failed"; + std::cerr << std::endl; +#endif + } + } + + delete inItem; + free(ptr); + return item; +} + void P3Hub::run() { @@ -89,8 +144,16 @@ void P3Hub::run() std::cerr << std::endl; #endif - - recvdQ.push_back(make_pair(it->first, item)); + if (mUseSerialiser) + { + item = SerialiserPass(item); + } + + /* serialiser might hav munched it. */ + if (item) + { + recvdQ.push_back(make_pair(it->first, item)); + } } } diff --git a/libretroshare/src/ft/pqitestor.h b/libretroshare/src/ft/pqitestor.h index 109398248..0c445acbd 100644 --- a/libretroshare/src/ft/pqitestor.h +++ b/libretroshare/src/ft/pqitestor.h @@ -67,13 +67,18 @@ class P3Hub: public RsThread { public: - P3Hub(); + P3Hub(uint32_t flags, RsSerialiser *rss); void addP3Pipe(std::string id, P3Pipe *, p3ConnectMgr *mgr); virtual void run(); private: + +RsItem* SerialiserPass(RsItem *inItem); + std::map mPeers; + RsSerialiser *mSerialiser; + bool mUseSerialiser; }; diff --git a/libretroshare/src/pqi/p3connmgr.cc b/libretroshare/src/pqi/p3connmgr.cc index 7287cdff1..7c3693320 100644 --- a/libretroshare/src/pqi/p3connmgr.cc +++ b/libretroshare/src/pqi/p3connmgr.cc @@ -63,7 +63,6 @@ const uint32_t MAX_UPNP_INIT = 10; /* seconds UPnP timeout */ * #define P3CONNMGR_NO_AUTO_CONNECTION 1 ***/ -#define CONN_DEBUG 1 const uint32_t P3CONNMGR_TCP_DEFAULT_DELAY = 2; /* 2 Seconds? is it be enough! */ const uint32_t P3CONNMGR_UDP_DHT_DELAY = DHT_NOTIFY_PERIOD + 60; /* + 1 minute for DHT POST */ diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 94024eae1..a19edf87b 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -540,21 +540,25 @@ int pqistreamer::handleincoming() std::string title = "Warning: Bad Packet Read"; - std::string msg; - msg += " **** WARNING **** \n"; - msg += "Retroshare has caught a BAD Packet Read"; - msg += "\n"; - msg += "This is normally caused by connecting to an"; - msg += " OLD version of Retroshare"; - msg += "\n"; - msg += "\n"; - msg += "Please get your friends to upgrade to the latest version"; - msg += "\n"; - msg += "\n"; - msg += "If you are sure the error was not caused by an old version"; - msg += "\n"; - msg += "Please report the problem to Retroshare's developers"; - msg += "\n"; + std::ostringstream msgout; + msgout << " **** WARNING **** \n"; + msgout << "Retroshare has caught a BAD Packet Read"; + msgout << "\n"; + msgout << "This is normally caused by connecting to an"; + msgout << " OLD version of Retroshare"; + msgout << "\n"; + msgout << "(M:" << maxlen << " B:" << blen << " E:" << extralen << ")\n"; + msgout << "\n"; + msgout << "\n"; + msgout << "Please get your friends to upgrade to the latest version"; + msgout << "\n"; + msgout << "\n"; + msgout << "If you are sure the error was not caused by an old version"; + msgout << "\n"; + msgout << "Please report the problem to Retroshare's developers"; + msgout << "\n"; + + std::string msg = msgout.str(); notify->AddSysMessage(0, RS_SYS_WARNING, title, msg); } bio->close(); @@ -580,13 +584,17 @@ int pqistreamer::handleincoming() std::string title = "Warning: Error Completing Read"; - std::string msg; - msg += " **** WARNING **** \n"; - msg += "Retroshare has experienced an unexpected Read ERROR"; - msg += "\n"; - msg += "Please contact the developers."; - msg += "\n"; + std::ostringstream msgout; + msgout << " **** WARNING **** \n"; + msgout << "Retroshare has experienced an unexpected Read ERROR"; + msgout << "\n"; + msgout << "(M:" << maxlen << " B:" << blen; + msgout << " E:" << extralen << " R:" << tmplen << ")\n"; + msgout << "\n"; + msgout << "Please contact the developers."; + msgout << "\n"; + std::string msg = msgout.str(); notify->AddSysMessage(0, RS_SYS_WARNING, title, msg); } bio->close(); diff --git a/libretroshare/src/rsiface/rstypes.h b/libretroshare/src/rsiface/rstypes.h index 2675c2d5b..fdc8c13f2 100644 --- a/libretroshare/src/rsiface/rstypes.h +++ b/libretroshare/src/rsiface/rstypes.h @@ -37,6 +37,22 @@ typedef std::string RsChanId; typedef std::string RsMsgId; typedef std::string RsAuthId; +const uint32_t FT_STATE_FAILED = 0x0000; +const uint32_t FT_STATE_OKAY = 0x0001; +const uint32_t FT_STATE_WAITING = 0x0002; +const uint32_t FT_STATE_DOWNLOADING = 0x0003; +const uint32_t FT_STATE_COMPLETE = 0x0004; + +class TransferInfo +{ + public: + /**** Need Some of these Fields ****/ + std::string peerId; + std::string name; /* if has alternative name? */ + double tfRate; /* kbytes */ + int status; /* FT_STATE_... */ +}; + class FileInfo { @@ -75,37 +91,13 @@ static const int kRsFiStatusDone = 2; double rank; int age; - /* Old FileTransferInfo Entries */ - public: - std::string source; - std::list peerIds; - int transfered; - double tfRate; /* kbytes */ - bool download; - int downloadStatus; /* 0 = Err, 1 = Ok, 2 = Done */ - - /* ENTRIES USED BY SFI *** - * - * path, - * fname, - * hash, - * size, - * avail, - * - * source? - * - */ - - + /* Transfer Stuff */ + uint64_t transfered; + double tfRate; /* in kbytes */ + uint32_t downloadStatus; /* 0 = Err, 1 = Ok, 2 = Done */ + std::list peers; }; -class FileTransferInfo: public FileInfo -{ - public: - FileTransferInfo() { return; } -}; - - std::ostream &operator<<(std::ostream &out, const FileInfo &info); diff --git a/libretroshare/src/rsserver/Makefile b/libretroshare/src/rsserver/Makefile index 495a64952..a846dc819 100644 --- a/libretroshare/src/rsserver/Makefile +++ b/libretroshare/src/rsserver/Makefile @@ -18,9 +18,9 @@ RSOBJ = p3peers.o \ p3face-startup.o \ p3face-msgs.o \ rsiface.o \ - p3files.o \ rstypes.o +# p3files.o \ # p3face-file.o \ # pqistrings.o \ # p3face-people.o diff --git a/libretroshare/src/rsserver/p3face-config.cc b/libretroshare/src/rsserver/p3face-config.cc index 9501ce73a..425ad96cc 100644 --- a/libretroshare/src/rsserver/p3face-config.cc +++ b/libretroshare/src/rsserver/p3face-config.cc @@ -155,7 +155,11 @@ int RsServer::UpdateAllConfig() void RsServer::ConfigFinalSave() { /* force saving of transfers */ +#ifdef USE_OLD_FT server->saveFileTransferStatus(); +#else + //ftserver->saveFileTransferStatus(); +#endif mAuthMgr->FinalSaveCertificates(); mConfigMgr->completeConfiguration(); diff --git a/libretroshare/src/rsserver/p3face-server.cc b/libretroshare/src/rsserver/p3face-server.cc index 87cb37c88..c26cd6135 100644 --- a/libretroshare/src/rsserver/p3face-server.cc +++ b/libretroshare/src/rsserver/p3face-server.cc @@ -26,7 +26,11 @@ #include "rsserver/p3face.h" + +#ifdef USE_OLD_FT #include "rsserver/p3files.h" // TMP FOR HACK! +#endif + #include "tcponudp/tou.h" #include @@ -117,9 +121,14 @@ void RsServer::run() /******************************** RUN SERVER *****************/ lockRsCore(); +#ifdef USE_OLD_FT int moreToTick = server -> tick(); +#else + int moreToTick = ftserver -> tick(); +#endif + #ifdef DEBUG_TICK - std::cerr << "RsServer::run() server->tick(): moreToTick: " << moreToTick << std::endl; + std::cerr << "RsServer::run() ftserver->tick(): moreToTick: " << moreToTick << std::endl; #endif unlockRsCore(); @@ -191,6 +200,8 @@ void RsServer::run() // currently Dummy Functions. //std::cerr << "RsServer::run() UpdateAllTransfers()" << std::endl; + +#ifdef USE_OLD_FT // // TMP HACK. p3Files *p3f = (p3Files *) rsFiles; @@ -198,6 +209,7 @@ void RsServer::run() { p3f -> UpdateAllTransfers(); } +#endif //std::cerr << "RsServer::run() "; //std::cerr << "UpdateRemotePeople()"<saveFileTransferStatus(); +#ifdef USE_OLD_FT + ftserver->saveFileTransferStatus(); +#else + //ftserver->saveFileTransferStatus(); +#endif /* see if we need to resave certs */ mAuthMgr->CheckSaveCertificates(); diff --git a/libretroshare/src/rsserver/p3face-startup.cc b/libretroshare/src/rsserver/p3face-startup.cc index e975add99..b9209571c 100644 --- a/libretroshare/src/rsserver/p3face-startup.cc +++ b/libretroshare/src/rsserver/p3face-startup.cc @@ -28,8 +28,19 @@ //#include #include "dbase/cachestrapper.h" -#include "server/ftfiler.h" -#include "server/filedexserver.h" +#ifdef USE_OLD_FT + #include "server/ftfiler.h" + #include "server/filedexserver.h" +#else + #include "ft/ftserver.h" + #include "ft/ftcontroller.h" +#endif + +/* global variable now points straight to + * ft/ code so variable defined here. + */ + +RsFiles *rsFiles = NULL; #include "pqi/pqipersongrp.h" #include "pqi/pqisslpersongrp.h" @@ -520,15 +531,18 @@ int RsServer::StartupRetroShare(RsInit *config) std::string certConfigFile = config->basedir.c_str(); std::string certNeighDir = config->basedir.c_str(); std::string emergencySaveDir = config->basedir.c_str(); + std::string emergencyPartialsDir = config->basedir.c_str(); if (certConfigFile != "") { certConfigFile += "/"; certNeighDir += "/"; emergencySaveDir += "/"; + emergencyPartialsDir += "/"; } certConfigFile += configConfFile; certNeighDir += configCertDir; emergencySaveDir += "Downloads"; + emergencyPartialsDir += "Partials"; /* if we've loaded an old format file! */ bool oldFormat = false; @@ -549,13 +563,14 @@ int RsServer::StartupRetroShare(RsInit *config) pqiNetAssistFirewall *mUpnpMgr = new upnphandler(); p3DhtMgr *mDhtMgr = new OpenDHTMgr(ownId, mConnMgr, config->basedir); - CacheStrapper *mCacheStrapper = new CacheStrapper(mAuthMgr, mConnMgr); - ftfiler *mCacheTransfer = new ftfiler(mCacheStrapper); - SecurityPolicy *none = secpolicy_create(); pqih = new pqisslpersongrp(none, flags); //pqih = new pqipersongrpDummy(none, flags); +#ifdef USE_OLD_FT + CacheStrapper *mCacheStrapper = new CacheStrapper(mAuthMgr, mConnMgr); + ftfiler *mCacheTransfer = new ftfiler(mCacheStrapper); + // filedex server. server = new filedexserver(); server->setConfigDir(config->basedir.c_str()); @@ -571,6 +586,25 @@ int RsServer::StartupRetroShare(RsInit *config) rsFiles = new p3Files(server, this, mAuthMgr); +#else +/****** New Ft Server **** !!! */ + ftserver = new ftServer(mAuthMgr, mConnMgr); + ftserver->setP3Interface(pqih); + ftserver->setConfigDirectory(config->basedir); + + ftserver->SetupFtServer(&(getNotify())); + CacheStrapper *mCacheStrapper = ftserver->getCacheStrapper(); + CacheTransfer *mCacheTransfer = ftserver->getCacheTransfer(); + + /* setup any extra bits (Default Paths) */ + ftserver->setPartialsDirectory(emergencyPartialsDir); + ftserver->setDownloadDirectory(emergencySaveDir); + + /* This should be set by config ... there is no default */ + //ftserver->setSharedDirectories(fileList); + + rsFiles = ftserver; +#endif mConfigMgr = new p3ConfigMgr(mAuthMgr, config->basedir, "rs-v0.4.cfg", "rs-v0.4.sgn"); mGeneralConfig = new p3GeneralConfig(); @@ -649,9 +683,22 @@ int RsServer::StartupRetroShare(RsInit *config) mConnMgr->addMonitor(ad); mConnMgr->addMonitor(msgSrv); + /* must also add the controller as a Monitor... + * a little hack to get it to work. + */ +#ifdef USE_OLD_FT +#else + mConnMgr->addMonitor(((ftController *) mCacheTransfer)); +#endif + + /**************************************************************************/ +#ifdef USE_OLD_FT mConfigMgr->addConfiguration("server.cfg", server); +#else + //mConfigMgr->addConfiguration("ftserver.cfg", ftserver); +#endif mConfigMgr->addConfiguration("peers.cfg", mConnMgr); mConfigMgr->addConfiguration("general.cfg", mGeneralConfig); mConfigMgr->addConfiguration("msgs.cfg", msgSrv); @@ -767,13 +814,12 @@ int RsServer::StartupRetroShare(RsInit *config) /* Start up Threads */ /**************************************************************************/ +#ifdef USE_OLD_FT server->StartupMonitor(); - mDhtMgr->start(); - - -#ifdef PQI_USE_CHANNELS - server->setP3Channel(pqih->getP3Channel()); +#else + ftserver->StartupThreads(); #endif + mDhtMgr->start(); // create loopback device, and add to pqisslgrp. diff --git a/libretroshare/src/rsserver/p3face.h b/libretroshare/src/rsserver/p3face.h index abc63a294..dd1ddf2dd 100644 --- a/libretroshare/src/rsserver/p3face.h +++ b/libretroshare/src/rsserver/p3face.h @@ -26,7 +26,8 @@ * */ -#include "server/filedexserver.h" +//#include "server/filedexserver.h" +#include "ft/ftserver.h"; //#include "pqi/pqissl.h" #include "pqi/p3cfgmgr.h" @@ -156,7 +157,8 @@ int UpdateAllConfig(); // The real Server Parts. - filedexserver *server; + //filedexserver *server; + ftServer *ftserver; p3ConnectMgr *mConnMgr; p3AuthMgr *mAuthMgr; diff --git a/libretroshare/src/rsserver/p3files.cc b/libretroshare/src/rsserver/p3files.cc index 191c808ba..bdf3a3ae2 100644 --- a/libretroshare/src/rsserver/p3files.cc +++ b/libretroshare/src/rsserver/p3files.cc @@ -32,7 +32,7 @@ #include #include -RsFiles *rsFiles = NULL; +//RsFiles *rsFiles = NULL; void p3Files::lockRsCore() { diff --git a/libretroshare/src/scripts/config-macosx.mk b/libretroshare/src/scripts/config-macosx.mk index 42f98b15e..940d96aa1 100644 --- a/libretroshare/src/scripts/config-macosx.mk +++ b/libretroshare/src/scripts/config-macosx.mk @@ -51,7 +51,8 @@ LIBDIR = $(RS_TOP_DIR)/lib LIBRS = $(LIBDIR)/libretroshare.a INCLUDE = -I $(RS_TOP_DIR) -CFLAGS = -Wall -O3 +#CFLAGS = -Wall -O3 +CFLAGS = -Wall -g # Flags for architecture builds. ifdef MAC_I386_BUILD diff --git a/libretroshare/src/scripts/config.mk b/libretroshare/src/scripts/config.mk index 9f22fa363..e7d11205f 100644 --- a/libretroshare/src/scripts/config.mk +++ b/libretroshare/src/scripts/config.mk @@ -4,8 +4,8 @@ ########################################################################### #Define OS. # -OS = Linux -#OS = MacOSX +#OS = Linux +OS = MacOSX #OS = Cygwin #OS = Win # MinGw. ########################################################################### diff --git a/libretroshare/src/util/rsthreads.cc b/libretroshare/src/util/rsthreads.cc index 841390c9f..fec0cdd7c 100644 --- a/libretroshare/src/util/rsthreads.cc +++ b/libretroshare/src/util/rsthreads.cc @@ -28,6 +28,14 @@ #include "rsthreads.h" #include /* for usleep() */ +/******* + * #define DEBUG_THREADS 1 + *******/ + +#ifdef DEBUG_THREADS + #include +#endif + extern "C" void* rsthread_init(void* p) { RsThread *thread = (RsThread *) p; @@ -79,6 +87,12 @@ void RsQueueThread::run() mLastWork = now; mLastSleep = (uint32_t) (mMinSleep + (mLastSleep - mMinSleep) / 2.0); +#ifdef DEBUG_THREADS + std::cerr << "RsQueueThread::run() done work: sleeping for: " << mLastSleep; + std::cerr << " ms"; + std::cerr << std::endl; +#endif + } else { @@ -91,6 +105,11 @@ void RsQueueThread::run() { mLastSleep = mMaxSleep; } +#ifdef DEBUG_THREADS + std::cerr << "RsQueueThread::run() no work: sleeping for: " << mLastSleep; + std::cerr << " ms"; + std::cerr << std::endl; +#endif } #ifdef WIN32 Sleep(mLastSleep);