From a70b1060051b228e781c14e2d0b4e382622f8a59 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Jan 2010 20:40:21 +0000 Subject: [PATCH] - added NETWORK_WIDE flag to remote dir model download - added file priority strategy based on which files are requested first. This provides: - equal file speed for files with equal (source,priority) - effective priority speed for file of same source but different priority - removed state variable load/save from turtle, as it's not needed anymore (FileRequest re-opens tunnels as needed) - manage availability per peer instead of per file type: direct peer ids always assume file availability, while turtle tunnels don't I still need to make the download queue work, and code this gui for it. git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@2133 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/ft/ftchunkmap.cc | 84 +++--- libretroshare/src/ft/ftchunkmap.h | 5 +- libretroshare/src/ft/ftcontroller.cc | 326 +++++++++++++-------- libretroshare/src/ft/ftcontroller.h | 54 ++-- libretroshare/src/ft/ftfilecreator.cc | 4 +- libretroshare/src/ft/ftfilecreator.h | 2 +- libretroshare/src/ft/ftfileprovider.cc | 1 + libretroshare/src/ft/ftserver.cc | 8 +- libretroshare/src/rsiface/rsturtle.h | 3 + libretroshare/src/rsiface/rstypes.h | 14 +- libretroshare/src/turtle/p3turtle.cc | 11 +- libretroshare/src/turtle/p3turtle.h | 2 +- retroshare-gui/src/gui/RemoteDirModel.cpp | 5 +- retroshare-gui/src/gui/TransfersDialog.cpp | 36 ++- retroshare-gui/src/gui/TransfersDialog.h | 6 +- 15 files changed, 352 insertions(+), 209 deletions(-) diff --git a/libretroshare/src/ft/ftchunkmap.cc b/libretroshare/src/ft/ftchunkmap.cc index 2fd96e9d0..42f28726a 100644 --- a/libretroshare/src/ft/ftchunkmap.cc +++ b/libretroshare/src/ft/ftchunkmap.cc @@ -4,6 +4,7 @@ #include #include #include +#include #include "ftchunkmap.h" #include @@ -35,10 +36,9 @@ void Chunk::getSlice(uint32_t size_hint,ftChunk& chunk) _offset += chunk.size ; } -ChunkMap::ChunkMap(uint64_t s,bool assume_availability) +ChunkMap::ChunkMap(uint64_t s) : _file_size(s), - _chunk_size(CHUNKMAP_FIXED_CHUNK_SIZE), // 1MB chunks - _assume_availability(assume_availability) + _chunk_size(CHUNKMAP_FIXED_CHUNK_SIZE) // 1MB chunks { uint64_t n = s/(uint64_t)_chunk_size ; if(s% (uint64_t)_chunk_size != 0) @@ -290,56 +290,66 @@ uint32_t ChunkMap::sizeOfChunk(uint32_t cid) const uint32_t ChunkMap::getAvailableChunk(uint32_t start_location,const std::string& peer_id,bool& map_is_too_old) { - // Very bold algorithm: checks for 1st availabe chunk for this peer starting - // from the given start location. + // Quite simple strategy: Check for 1st availabe chunk for this peer starting from the given start location. + // std::map::iterator it(_peers_chunks_availability.find(peer_id)) ; SourceChunksInfo *peer_chunks = NULL; - // Do we have records for this file source ? + // Do we have a chunk map for this file source ? + // - if yes, we use it + // - if no, + // - if availability is assumed, let's build a plain chunkmap for it + // - otherwise, refuse the transfer, but still ask for the chunkmap // - if(!_assume_availability) + // We first test whether the source has a record of not. If not, we fill a new record. + // For availability sources we fill it plain, otherwise, we fill it blank. + // + if(it == _peers_chunks_availability.end()) { - if(it == _peers_chunks_availability.end()) - { - SourceChunksInfo& pchunks(_peers_chunks_availability[peer_id]) ; + SourceChunksInfo& pchunks(_peers_chunks_availability[peer_id]) ; - // Ok, we don't have the info, so two cases: - // - we are the actual source, so we can safely init the map to a full map - // - we are not the source, so we init it with an empty map, and set the time stamp to 0. - // - if(peer_id == rsPeers->getOwnId()) - { - pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),~(uint32_t)0 ) ; - pchunks.TS = 0 ; - pchunks.is_full = true ; - } - else - { - pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),0 ) ; - pchunks.TS = 0 ; - pchunks.is_full = false ; - } + bool assume_availability = !rsTurtle->isTurtlePeer(peer_id) ; - it = _peers_chunks_availability.find(peer_id) ; - } - peer_chunks = &(it->second) ; - - // If the info is too old, we ask for a new one. When the map is full, we ask 10 times less, as it's probably not - // useful to get a new map that will also be full, but because we need to be careful not to mislead information, - // we still keep asking. + // Ok, we don't have the info, so two cases: + // - peer_id is a not a turtle peer, so he is considered having the full file source, so we init with a plain chunkmap + // - otherwise, a source map needs to be obtained, so we init with a blank chunkmap // - time_t now = time(NULL) ; - map_is_too_old = (int)now - (int)peer_chunks->TS > (int)SOURCE_CHUNK_MAP_UPDATE_PERIOD*(1+9*peer_chunks->is_full) ; + if(assume_availability) + { + pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),~(uint32_t)0 ) ; + pchunks.TS = 0 ; + pchunks.is_full = true ; + } + else + { + pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),0 ) ; + pchunks.TS = 0 ; + pchunks.is_full = false ; + } - // We will re-ask but not now seconds. + it = _peers_chunks_availability.find(peer_id) ; + } + peer_chunks = &(it->second) ; + + // If the info is too old, we ask for a new one. When the map is full, we ask 10 times less, as it's probably not + // useful to get a new map that will also be full, but because we need to be careful not to mislead information, + // we still keep asking. + // + time_t now = time(NULL) ; + + if((!peer_chunks->is_full) && ((int)now - (int)peer_chunks->TS > (int)SOURCE_CHUNK_MAP_UPDATE_PERIOD)) + { + map_is_too_old = true ;// We will re-ask but not before some seconds. peer_chunks->TS = now ; } + else + map_is_too_old = false ;// the map is not too old for(unsigned int i=0;i<_map.size();++i) { uint32_t j = (start_location+i)%(int)_map.size() ; // index of the chunk - if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && (_assume_availability || peer_chunks->cmap[j])) + if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && (peer_chunks->is_full || peer_chunks->cmap[j])) { #ifdef DEBUG_FTCHUNK std::cerr << "ChunkMap::getAvailableChunk: returning chunk " << j << " for peer " << peer_id << std::endl; diff --git a/libretroshare/src/ft/ftchunkmap.h b/libretroshare/src/ft/ftchunkmap.h index c19aa2b7e..ab7148e3a 100644 --- a/libretroshare/src/ft/ftchunkmap.h +++ b/libretroshare/src/ft/ftchunkmap.h @@ -27,6 +27,8 @@ // This class handles a slice of a chunk of arbitrary uint32_t size, at the level of ftFileCreator +class ftController ; + class ftChunk { public: @@ -90,7 +92,7 @@ class ChunkMap /// Constructor. Decides what will be the size of chunks and how many there will be. - ChunkMap(uint64_t file_size,bool assume_sources_availability) ; + ChunkMap(uint64_t file_size) ; /// constructor from saved map info ChunkMap(uint64_t file_size,const std::vector& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ; @@ -169,7 +171,6 @@ class ChunkMap std::vector _map ; //! vector of chunk state over the whole file std::map _peers_chunks_availability ; //! what does each source peer have uint64_t _total_downloaded ; //! completion for the file - bool _assume_availability ; //! set for transfers where the source is always available }; diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index d74112e87..3a7628970 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -62,7 +62,8 @@ static const uint32_t INACTIVE_CHUNKS_CHECK_DELAY = 60 ; // save transfer progre ftFileControl::ftFileControl() :mTransfer(NULL), mCreator(NULL), - mState(0), mSize(0), mFlags(0) + mState(0), mSize(0), mFlags(0), + mPriority(PRIORITY_NORMAL) { return; } @@ -73,7 +74,8 @@ ftFileControl::ftFileControl(std::string fname, ftFileCreator *fc, ftTransferModule *tm, uint32_t cb) :mName(fname), mCurrentPath(tmppath), mDestination(dest), mTransfer(tm), mCreator(fc), mState(0), mHash(hash), - mSize(size), mFlags(flags), mDoCallback(false), mCallbackCode(cb) + mSize(size), mFlags(flags), mDoCallback(false), mCallbackCode(cb), + mPriority(PRIORITY_NORMAL) // default priority to normal { if (cb) mDoCallback = true; @@ -194,6 +196,8 @@ void ftController::run() time_t now = time(NULL) ; if((int)now - (int)last_save_time > (int)SAVE_TRANSFERS_DELAY) { + cleanCacheDownloads() ; + IndicateConfigChanged() ; last_save_time = now ; } @@ -217,65 +221,160 @@ void ftController::run() } } - /* tick the transferModules */ - std::list done; - std::list::iterator it; + tickTransfers() ; + { - RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ + RsStackMutex stack2(doneMutex); - std::map::iterator it; - std::map currentDownloads = *(&mDownloads); - for(it = currentDownloads.begin(); it != currentDownloads.end(); it++) - { - -#ifdef CONTROL_DEBUG - std::cerr << "\tTicking: " << it->first; - std::cerr << std::endl; -#endif - - if (it->second.mTransfer) - { -#ifdef CONTROL_DEBUG - std::cerr << "\tTicking mTransfer: " << (void*)it->second.mTransfer; - std::cerr << std::endl; -#endif - (it->second.mTransfer)->tick(); - - - //check if a cache file is downloaded, if the case, timeout the transfer after TIMOUT_CACHE_FILE_TRANSFER - if ((it->second).mFlags & RS_FILE_HINTS_CACHE) { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::run() cache transfer found. age of this tranfer is :" << (int)(time(NULL) - (it->second).mCreateTime); - std::cerr << std::endl; -#endif - if ((time(NULL) - (it->second).mCreateTime) > TIMOUT_CACHE_FILE_TRANSFER) { -#ifdef CONTROL_DEBUG - std::cerr << "ftController::run() cache transfer to old. Cancelling transfer. Hash :" << (it->second).mHash << ", time=" << (it->second).mCreateTime << ", now = " << time(NULL) ; - std::cerr << std::endl; -#endif - this->FileCancel((it->second).mHash); - } - } - - } -#ifdef CONTROL_DEBUG - else - std::cerr << "No mTransfer for this hash." << std::endl ; -#endif - } + for(std::list::iterator it(mDone.begin()); it != mDone.end(); it++) + completeFile(*it); + + mDone.clear(); } - - RsStackMutex stack2(doneMutex); - for(it = mDone.begin(); it != mDone.end(); it++) - { - completeFile(*it); - } - mDone.clear(); } } +void ftController::tickTransfers() +{ + // 1 - sort modules into arrays according to priority + +// if(mPrioritiesChanged) + + RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ + std::cerr << "ticking transfers." << std::endl ; + mPriorityTab = std::vector >(3,std::vector()) ; + + for(std::map::iterator it(mDownloads.begin()); it != mDownloads.end(); it++) + mPriorityTab[it->second.mPriority].push_back(it->second.mTransfer) ; + + // 2 - tick arrays with a probability proportional to priority + + // 2.1 - decide based on probability, which category of files we handle. + + static const float HIGH_PRIORITY_PROB = 0.60 ; + static const float NORMAL_PRIORITY_PROB = 0.25 ; + static const float LOW_PRIORITY_PROB = 0.15 ; + static const float SUSP_PRIORITY_PROB = 0.00 ; + + std::cerr << "Priority tabs: " ; + std::cerr << "Low (" << mPriorityTab[PRIORITY_LOW ].size() << ") " ; + std::cerr << "Normal (" << mPriorityTab[PRIORITY_NORMAL].size() << ") " ; + std::cerr << "High (" << mPriorityTab[PRIORITY_HIGH ].size() << ") " ; + std::cerr << std::endl ; + +// float probs[3] = { (!mPriorityTab[PRIORITY_LOW ].empty())* LOW_PRIORITY_PROB, +// (!mPriorityTab[PRIORITY_NORMAL].empty())*NORMAL_PRIORITY_PROB, +// (!mPriorityTab[PRIORITY_HIGH ].empty())* HIGH_PRIORITY_PROB } ; +// +// float total = probs[0]+probs[1]+probs[2] ; +// float cumul_probs[3] = { probs[0], probs[0]+probs[1], probs[0]+probs[1]+probs[2] } ; +// +// float r = rand()/(float)RAND_MAX * total; +// int chosen ; +// +// if(total == 0.0) +// return ; +// +// if(r < cumul_probs[0]) +// chosen = 0 ; +// else if(r < cumul_probs[1]) +// chosen = 1 ; +// else +// chosen = 2 ; +// +// std::cerr << "chosen: " << chosen << std::endl ; + + /* tick the transferModules */ + + // start anywhere in the chosen list of transfers, so that not to favor any special transfer + // + + for(int chosen=2;chosen>=0;--chosen) + if(!mPriorityTab[chosen].empty()) + { + int start = rand() % mPriorityTab[chosen].size() ; + + for(int i=0;i<(int)mPriorityTab[chosen].size();++i) + mPriorityTab[chosen][(i+start)%(int)mPriorityTab[chosen].size()]->tick() ; + } + +// { +// +//#ifdef CONTROL_DEBUG +// std::cerr << "\tTicking: " << it->first; +// std::cerr << std::endl; +//#endif +// +// if (it->second.mTransfer) +// { +//#ifdef CONTROL_DEBUG +// std::cerr << "\tTicking mTransfer: " << (void*)it->second.mTransfer; +// std::cerr << std::endl; +//#endif +// (it->second.mTransfer)->tick(); +// } +//#ifdef CONTROL_DEBUG +// else +// std::cerr << "No mTransfer for this hash." << std::endl ; +//#endif +// } +// } +} + +bool ftController::getPriority(const std::string& hash,DwlPriority& p) +{ + RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ + + std::map::iterator it(mDownloads.find(hash)) ; + + if(it != mDownloads.end()) + { + p = it->second.mPriority ; + return true ; + } + else + return false ; +} +void ftController::setPriority(const std::string& hash,DwlPriority p) +{ + RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ + + std::map::iterator it(mDownloads.find(hash)) ; + + if(it != mDownloads.end()) + it->second.mPriority = p ; +} + +void ftController::cleanCacheDownloads() +{ + std::vector toCancel ; + + { + RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ + + for(std::map::iterator it(mDownloads.begin());it!=mDownloads.end();++it) + if ((it->second).mFlags & RS_FILE_HINTS_CACHE) //check if a cache file is downloaded, if the case, timeout the transfer after TIMOUT_CACHE_FILE_TRANSFER + { +#ifdef CONTROL_DEBUG + std::cerr << "ftController::run() cache transfer found. age of this tranfer is :" << (int)(time(NULL) - (it->second).mCreateTime); + std::cerr << std::endl; +#endif + if ((time(NULL) - (it->second).mCreateTime) > TIMOUT_CACHE_FILE_TRANSFER) + { +#ifdef CONTROL_DEBUG + std::cerr << "ftController::run() cache transfer to old. Cancelling transfer. Hash :" << (it->second).mHash << ", time=" << (it->second).mCreateTime << ", now = " << time(NULL) ; + std::cerr << std::endl; +#endif + toCancel.push_back((it->second).mHash); + } + } + } + + for(uint32_t i=0;igetOwnId(); uint32_t rate = 0; if (flags & RS_FILE_HINTS_BACKGROUND) - { rate = FT_CNTRL_SLOW_RATE; - } else - { rate = FT_CNTRL_STANDARD_RATE; - } /* First check if the file is already being downloaded.... * This is important as some guis request duplicate files regularly. @@ -844,15 +939,8 @@ bool ftController::FileRequest(std::string fname, std::string hash, if(flags & RS_FILE_HINTS_NETWORK_WIDE) mTurtle->monitorFileTunnels(fname,hash,size) ; - else - { - std::cerr << "Warning: no flags supplied. Assuming availability. This is probably a bug." << std::endl ; - flags |= RS_FILE_HINTS_ASSUME_AVAILABILITY ; - } - bool assume_source_availability = (flags & RS_FILE_HINTS_ASSUME_AVAILABILITY) > 0 ; - - ftFileCreator *fc = new ftFileCreator(savepath, size, hash, 0,assume_source_availability); + ftFileCreator *fc = new ftFileCreator(savepath, size, hash, 0); ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this); /* add into maps */ @@ -894,9 +982,7 @@ bool ftController::FileRequest(std::string fname, std::string hash, return true; } - -bool ftController::setPeerState(ftTransferModule *tm, std::string id, - uint32_t maxrate, bool online) +bool ftController::setPeerState(ftTransferModule *tm, std::string id, uint32_t maxrate, bool online) { if (id == mConnMgr->getOwnId()) { @@ -935,7 +1021,7 @@ bool ftController::setChunkStrategy(const std::string& hash,FileChunksInfo::Chun if (mit==mDownloads.end()) { #ifdef CONTROL_DEBUG - std::cerr<<"ftController::FileCancel file is not found in mDownloads"<::iterator mit; - mit=mDownloads.find(hash); - if (mit==mDownloads.end()) + { + RsStackMutex mtx(ctrlMutex) ; + + std::map::iterator mit; + mit=mDownloads.find(hash); + if (mit==mDownloads.end()) + { #ifdef CONTROL_DEBUG - std::cerr<<"ftController::FileCancel file is not found in mDownloads"<second).mCreator->finished()) - { + /* check if finished */ + if ((mit->second).mCreator->finished()) + { #ifdef CONTROL_DEBUG - std::cerr << "ftController:FileCancel(" << hash << ")"; - std::cerr << " Transfer Already finished"; - std::cerr << std::endl; + std::cerr << "ftController:FileCancel(" << hash << ")"; + std::cerr << " Transfer Already finished"; + std::cerr << std::endl; - std::cerr << "FileSize: "; - std::cerr << (mit->second).mCreator->getFileSize(); - std::cerr << " and Recvd: "; - std::cerr << (mit->second).mCreator->getRecvd(); + std::cerr << "FileSize: "; + std::cerr << (mit->second).mCreator->getFileSize(); + std::cerr << " and Recvd: "; + std::cerr << (mit->second).mCreator->getRecvd(); #endif - return false; - } + return false; + } - /*find the point to transfer module*/ - ftTransferModule* ft=(mit->second).mTransfer; - ft->cancelTransfer(); + /*find the point to transfer module*/ + ftTransferModule* ft=(mit->second).mTransfer; + ft->cancelTransfer(); - ftFileControl *fc = &(mit->second); - mDataplex->removeTransferModule(fc->mTransfer->hash()); + ftFileControl *fc = &(mit->second); + mDataplex->removeTransferModule(fc->mTransfer->hash()); - if (fc->mTransfer) - { - delete fc->mTransfer; - fc->mTransfer = NULL; - } + if (fc->mTransfer) + { + delete fc->mTransfer; + fc->mTransfer = NULL; + } - if (fc->mCreator) - { - delete fc->mCreator; - fc->mCreator = NULL; - } + if (fc->mCreator) + { + delete fc->mCreator; + fc->mCreator = NULL; + } - /* delete the temporary file */ - if (0 == remove(fc->mCurrentPath.c_str())) - { + /* delete the temporary file */ + if (0 == remove(fc->mCurrentPath.c_str())) + { #ifdef CONTROL_DEBUG - std::cerr << "ftController::FileCancel() remove temporary file "; - std::cerr << fc->mCurrentPath; - std::cerr << std::endl; + std::cerr << "ftController::FileCancel() remove temporary file "; + std::cerr << fc->mCurrentPath; + std::cerr << std::endl; #endif - } - else - { + } + else + { #ifdef CONTROL_DEBUG - std::cerr << "ftController::FileCancel() fail to remove file "; - std::cerr << fc->mCurrentPath; - std::cerr << std::endl; + std::cerr << "ftController::FileCancel() fail to remove file "; + std::cerr << fc->mCurrentPath; + std::cerr << std::endl; #endif - } + } - mDownloads.erase(mit); + mDownloads.erase(mit); + } IndicateConfigChanged(); /* completed transfer -> save */ return true; @@ -1196,6 +1287,7 @@ bool ftController::FileDetails(std::string hash, FileInfo &info) info.fname = it->second.mName; info.flags = it->second.mFlags; info.path = RsDirUtil::removeTopDir(it->second.mDestination); /* remove fname */ + info.priority = it->second.mPriority ; /* get list of sources from transferModule */ std::list peerIds; @@ -1396,7 +1488,7 @@ bool ftController::RequestCacheFile(RsPeerId id, std::string path, std::string h std::list ids; ids.push_back(id); - FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH | RS_FILE_HINTS_ASSUME_AVAILABILITY, ids); + FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids); return true; } diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index 834227c33..f8a726e08 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -66,25 +66,26 @@ class ftFileControl { public: - enum {DOWNLOADING,COMPLETED,ERROR_COMPLETION}; + enum {DOWNLOADING,COMPLETED,ERROR_COMPLETION}; - ftFileControl(); - ftFileControl(std::string fname, std::string tmppath, std::string dest, - uint64_t size, std::string hash, uint32_t flags, - ftFileCreator *fc, ftTransferModule *tm, uint32_t cb_flags); + ftFileControl(); + ftFileControl(std::string fname, std::string tmppath, std::string dest, + uint64_t size, std::string hash, uint32_t flags, + ftFileCreator *fc, ftTransferModule *tm, uint32_t cb_flags); - std::string mName; - std::string mCurrentPath; /* current full path (including name) */ - std::string mDestination; /* final full path (including name) */ - ftTransferModule * mTransfer; - ftFileCreator * mCreator; - uint32_t mState; - std::string mHash; - uint64_t mSize; - uint32_t mFlags; - bool mDoCallback; - uint32_t mCallbackCode; - time_t mCreateTime; + std::string mName; + std::string mCurrentPath; /* current full path (including name) */ + std::string mDestination; /* final full path (including name) */ + ftTransferModule * mTransfer; + ftFileCreator * mCreator; + uint32_t mState; + std::string mHash; + uint64_t mSize; + uint32_t mFlags; + bool mDoCallback; + uint32_t mCallbackCode; + time_t mCreateTime; + DwlPriority mPriority ; }; class ftPendingRequest @@ -143,6 +144,9 @@ bool FileClearCompleted(); bool FlagFileComplete(std::string hash); bool getFileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info); +bool getPriority(const std::string& hash,DwlPriority& p); +void setPriority(const std::string& hash,DwlPriority p); + /* get Details of File Transfers */ bool FileDownloads(std::list &hashs); @@ -159,11 +163,19 @@ bool moveFile(const std::string& source,const std::string& dest) ; /********************** Cache Transfer *************************/ /***************************************************************/ +/// Returns true is full source availability can be assumed for this peer. +/// +bool assumeAvailability(const std::string& peer_id) const ; + protected: virtual bool RequestCacheFile(RsPeerId id, std::string path, std::string hash, uint64_t size); virtual bool CancelCacheFile(RsPeerId id, std::string path, std::string hash, uint64_t size); +void cleanCacheDownloads() ; +void tickTransfers() ; + + /***************************************************************/ /********************** Controller Access **********************/ /***************************************************************/ @@ -205,9 +217,7 @@ bool setPeerState(ftTransferModule *tm, std::string id, std::list incomingQueue; std::map mCompleted; - - - std::map mDownloads; + std::map mDownloads; //std::map mTransfers; //std::map mFileCreators; @@ -233,6 +243,10 @@ bool setPeerState(ftTransferModule *tm, std::string id, /* share incoming directory */ bool mShareDownloadDir; + + // priority handling + // + std::vector< std::vector > mPriorityTab ; }; #endif diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index 043506fd5..75016126c 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -15,8 +15,8 @@ * ***********************************************************/ -ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash, uint64_t recvd,bool assume_sources_availability) - : ftFileProvider(path,size,hash), chunkMap(size,assume_sources_availability) +ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash, uint64_t recvd) + : ftFileProvider(path,size,hash), chunkMap(size) { /* * FIXME any inits to do? diff --git a/libretroshare/src/ft/ftfilecreator.h b/libretroshare/src/ft/ftfilecreator.h index 161635ef0..c5c79077c 100644 --- a/libretroshare/src/ft/ftfilecreator.h +++ b/libretroshare/src/ft/ftfilecreator.h @@ -40,7 +40,7 @@ class ftFileCreator: public ftFileProvider { public: - ftFileCreator(std::string savepath, uint64_t size, std::string hash, uint64_t recvd,bool assume_sources_availability=false); + ftFileCreator(std::string savepath, uint64_t size, std::string hash, uint64_t recvd); ~ftFileCreator(); diff --git a/libretroshare/src/ft/ftfileprovider.cc b/libretroshare/src/ft/ftfileprovider.cc index b3da9f4b7..ec7d57b6a 100644 --- a/libretroshare/src/ft/ftfileprovider.cc +++ b/libretroshare/src/ft/ftfileprovider.cc @@ -10,6 +10,7 @@ static const time_t UPLOAD_CHUNK_MAPS_TIME = 30 ; // time to ask for a new chunk ftFileProvider::ftFileProvider(std::string path, uint64_t size, std::string hash) : mSize(size), hash(hash), file_name(path), fd(NULL),transfer_rate(0),total_size(0) { + clients_chunk_maps.clear(); #ifdef DEBUG_FT_FILE_PROVIDER std::cout << "Creating file provider for " << hash << std::endl ; #endif diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 7e0457f39..17a779043 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -25,6 +25,7 @@ #include "util/rsdebug.h" #include "util/rsdir.h" +#include "rsiface/rstypes.h" const int ftserverzone = 29539; #include "ft/ftserver.h" @@ -249,7 +250,7 @@ bool ftServer::FileRequest(std::string fname, std::string hash, uint64_t size, s if(mFtController->alreadyHaveFile(hash)) return false ; - const DwlDetails details(fname, hash, size, dest, flags, srcIds, Normal); + const DwlDetails details(fname, hash, size, dest, flags, srcIds, PRIORITY_NORMAL); mFtDwlQueue->insertDownload(details); return true ; @@ -278,13 +279,14 @@ bool ftServer::FileClearCompleted() /* Control of Downloads Priority. */ bool ftServer::changePriority(const std::string hash, int priority) { - return mFtDwlQueue->changePriority(hash, (DwlPriority) priority); + mFtController->setPriority(hash, (DwlPriority) priority); + return true ; } bool ftServer::getPriority(const std::string hash, int & priority) { DwlPriority _priority; - int ret = mFtDwlQueue->getPriority(hash, _priority); + int ret = mFtController->getPriority(hash, _priority); if (ret) { priority = _priority; } diff --git a/libretroshare/src/rsiface/rsturtle.h b/libretroshare/src/rsiface/rsturtle.h index 61cd0ea21..2194bfaaf 100644 --- a/libretroshare/src/rsiface/rsturtle.h +++ b/libretroshare/src/rsiface/rsturtle.h @@ -92,6 +92,9 @@ class RsTurtle // Get info from the turtle router. I use std strings to hide the internal structs. virtual void getInfo(std::vector >&,std::vector >&, std::vector >&,std::vector >&) const = 0; + + // Convenience function. + virtual bool isTurtlePeer(const std::string& peer_id) const = 0 ; protected: FileSharingStrategy _sharing_strategy ; diff --git a/libretroshare/src/rsiface/rstypes.h b/libretroshare/src/rsiface/rstypes.h index 77d5ceba9..75c2a1764 100644 --- a/libretroshare/src/rsiface/rstypes.h +++ b/libretroshare/src/rsiface/rstypes.h @@ -57,6 +57,12 @@ class TransferInfo int status; /* FT_STATE_... */ }; +enum DwlPriority { PRIORITY_LOW = 0x00, + PRIORITY_NORMAL = 0x01, + PRIORITY_HIGH = 0x02, + PRIORITY_AUTO = 0x03 +}; + class FileInfo { @@ -101,6 +107,7 @@ class FileInfo uint32_t downloadStatus; /* 0 = Err, 1 = Ok, 2 = Done */ std::list peers; + DwlPriority priority ; time_t lastTS; }; @@ -247,13 +254,6 @@ class FileDetail uint32_t rank; }; -enum DwlPriority { Low = 0, Normal, High, Auto }; - -// Macro to read a bits array for compressed chunk maps -// -//#define COMPRESSED_MAP_READ(A,j) (A[j >> 5] & (1 << (j & 0x11111))) -//#define COMPRESSED_MAP_WRITE(A,j,x) (A[j >> 5] |= (1 << (j & 0x11111))) - class CompressedChunkMap ; class FileChunksInfo diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 74f073368..681436df2 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -424,7 +424,7 @@ std::list p3turtle::saveList(bool& cleanup) #endif cleanup = true ; std::list lst ; - +#ifdef TO_REMOVE RsTurtleSearchResultItem *item = new RsTurtleSearchResultItem ; item->PeerId("") ; @@ -441,11 +441,13 @@ std::list p3turtle::saveList(bool& cleanup) item->result.push_back(finfo) ; } lst.push_back(item) ; - +#endif return lst ; } + bool p3turtle::loadList(std::list load) { +#ifdef TO_REMOVE #ifdef P3TURTLE_DEBUG std::cerr << "p3turtle: loading list..." << std::endl ; #endif @@ -468,6 +470,7 @@ bool p3turtle::loadList(std::list load) } delete item ; } +#endif return true ; } @@ -797,8 +800,10 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item) case RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST: handleRecvFileMapRequest(dynamic_cast(item)) ; break ; default: - std::cerr << "Unknown packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ; + std::cerr << "WARNING: Unknown packet type received: id=" << (void*)(item->PacketSubType()) << ". Is somebody trying to DOS you ?" << std::endl ; +#ifdef P3TURTLE_DEBUG exit(-1) ; +#endif } delete item ; diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index 341f1777c..05a7c870c 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -263,7 +263,7 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f /************* Communication with ftserver *******************/ /// Does the turtle router manages tunnels to this peer ? (this is not a /// real id, but a fake one, that the turtle router is capable of connecting with a tunnel id). - bool isTurtlePeer(const std::string& peer_id) const ; + virtual bool isTurtlePeer(const std::string& peer_id) const ; /// Examines the peer id, finds the turtle tunnel in it, and respond yes if the tunnel is ok and operational. bool isOnline(const std::string& peer_id) const ; diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 014cf7b45..df50e26aa 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -23,6 +23,7 @@ #include "RemoteDirModel.h" #include "rsiface/rsfiles.h" +#include "rsiface/rstypes.h" #include #include @@ -867,7 +868,7 @@ void RemoteDirModel::downloadSelected(QModelIndexList list) std::list srcIds; srcIds.push_back(details.id); rsFiles -> FileRequest(details.name, details.hash, - details.count, "", 0, srcIds); + details.count, "", RS_FILE_HINTS_NETWORK_WIDE, srcIds); } /* if it is a dir, copy all files included*/ else if (details.type == DIR_TYPE_DIR) @@ -888,7 +889,7 @@ void RemoteDirModel::downloadDirectory(const DirDetails & dirDetails, int prefix QString cleanPath = QDir::cleanPath((rsFiles->getDownloadDirectory() + "/" + dirDetails.path.substr(prefixLen)).c_str()); srcIds.push_back(dirDetails.id); - rsFiles->FileRequest(dirDetails.name, dirDetails.hash, dirDetails.count, cleanPath.toStdString(), 0, srcIds); + rsFiles->FileRequest(dirDetails.name, dirDetails.hash, dirDetails.count, cleanPath.toStdString(), RS_FILE_HINTS_NETWORK_WIDE, srcIds); } else if (dirDetails.type & DIR_TYPE_DIR) { diff --git a/retroshare-gui/src/gui/TransfersDialog.cpp b/retroshare-gui/src/gui/TransfersDialog.cpp index 923433410..e53547b98 100644 --- a/retroshare-gui/src/gui/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/TransfersDialog.cpp @@ -457,19 +457,19 @@ TransfersDialog::~TransfersDialog() } -int TransfersDialog::addItem(QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, - QString sources, QString status, QString priority, qlonglong completed, qlonglong remaining) +int TransfersDialog::addItem(const QString& symbol, const QString& name, const QString& coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, + const QString& sources, const QString& status, const QString& priority, qlonglong completed, qlonglong remaining) { int row; QString sl; //QIcon icon(symbol); - name.insert(0, " "); + //name.insert(0, " "); //sl.sprintf("%d / %d", seeds, leechs); row = DLListModel->rowCount(); DLListModel->insertRow(row); //DLListModel->setData(DLListModel->index(row, NAME), QVariant((QIcon)icon), Qt::DecorationRole); - DLListModel->setData(DLListModel->index(row, NAME), QVariant((QString)name), Qt::DisplayRole); + DLListModel->setData(DLListModel->index(row, NAME), QVariant((QString)" "+name), Qt::DisplayRole); DLListModel->setData(DLListModel->index(row, SIZE), QVariant((qlonglong)fileSize)); DLListModel->setData(DLListModel->index(row, COMPLETED), QVariant((qlonglong)completed)); DLListModel->setData(DLListModel->index(row, DLSPEED), QVariant((double)dlspeed)); @@ -534,7 +534,7 @@ int TransfersDialog::addItem(QString symbol, QString name, QString coreID, qlong return row; } -bool TransfersDialog::addPeerToItem(int row, QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining) +bool TransfersDialog::addPeerToItem(int row, const QString& symbol, const QString& name, const QString& coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, const QString& sources, const QString& status, qlonglong completed, qlonglong remaining) { QStandardItem *dlItem = DLListModel->item(row); if (!dlItem) return false; @@ -542,10 +542,10 @@ bool TransfersDialog::addPeerToItem(int row, QString symbol, QString name, QStri //set this false if you want to expand on double click dlItem->setEditable(false); - name.insert(0, " "); + //name.insert(0, " "); QList items; - QStandardItem *i1 = new QStandardItem(); i1->setData(QVariant((QString)name), Qt::DisplayRole); + QStandardItem *i1 = new QStandardItem(); i1->setData(QVariant((QString)" "+name), Qt::DisplayRole); QStandardItem *i2 = new QStandardItem(); i2->setData(QVariant((qlonglong)fileSize), Qt::DisplayRole); QStandardItem *i3 = new QStandardItem(); i3->setData(QVariant((qlonglong)completed), Qt::DisplayRole); QStandardItem *i4 = new QStandardItem(); i4->setData(QVariant((double)dlspeed), Qt::DisplayRole); @@ -587,16 +587,16 @@ bool TransfersDialog::addPeerToItem(int row, QString symbol, QString name, QStri } -int TransfersDialog::addUploadItem(QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, QString source, QString status, qlonglong completed, qlonglong remaining) +int TransfersDialog::addUploadItem(const QString& symbol, const QString& name, const QString& coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, const QString& source, const QString& status, qlonglong completed, qlonglong remaining) { int row; QString sl; //QIcon icon(symbol); - name.insert(0, " "); + //name.insert(0, " "); row = ULListModel->rowCount(); ULListModel->insertRow(row); - ULListModel->setData(ULListModel->index(row, UNAME), QVariant((QString)name), Qt::DisplayRole); + ULListModel->setData(ULListModel->index(row, UNAME), QVariant((QString)" "+name), Qt::DisplayRole); ULListModel->setData(ULListModel->index(row, USIZE), QVariant((qlonglong)fileSize)); ULListModel->setData(ULListModel->index(row, UTRANSFERRED), QVariant((qlonglong)completed)); ULListModel->setData(ULListModel->index(row, ULSPEED), QVariant((double)dlspeed)); @@ -775,7 +775,21 @@ void TransfersDialog::insertTransfers() status = tr("Unknown"); break; } - priority = ""; /* for already downloading files */ + switch (info.priority) { + case 0: + priority = tr("Low"); + break; + case 1: + priority = tr("Normal"); + break; + case 2: + priority = tr("High"); + break; + default: + priority = tr("Auto"); + break; + } + completed = info.transfered; remaining = (info.size - info.transfered) / (info.tfRate * 1024.0); diff --git a/retroshare-gui/src/gui/TransfersDialog.h b/retroshare-gui/src/gui/TransfersDialog.h index d9b1bf371..039779695 100644 --- a/retroshare-gui/src/gui/TransfersDialog.h +++ b/retroshare-gui/src/gui/TransfersDialog.h @@ -168,11 +168,11 @@ class TransfersDialog : public RsAutoUpdatePage Ui::TransfersDialog ui; public slots: - int addItem(QString symbol, QString name, QString coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, QString priority, qlonglong completed, qlonglong remaining); - bool addPeerToItem(int row, QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining); + int addItem(const QString& symbol, const QString& name, const QString& coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, const QString& sources, const QString& status, const QString& priority, qlonglong completed, qlonglong remaining); + bool addPeerToItem(int row, const QString& symbol, const QString& name, const QString& coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, const QString& sources, const QString& status, qlonglong completed, qlonglong remaining); void delItem(int row); - int addUploadItem(QString symbol, QString name, QString coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining); + int addUploadItem(const QString& symbol, const QString& name, const QString& coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, const QString& sources, const QString& status, qlonglong completed, qlonglong remaining); void delUploadItem(int row); void editItem(int row, int column, QVariant data);