diff --git a/libretroshare/src/ft/ftchunkmap.cc b/libretroshare/src/ft/ftchunkmap.cc index 554643b7e..38b52ccb4 100644 --- a/libretroshare/src/ft/ftchunkmap.cc +++ b/libretroshare/src/ft/ftchunkmap.cc @@ -31,8 +31,10 @@ void Chunk::getSlice(uint32_t size_hint,ftChunk& chunk) _offset += chunk.size ; } -ChunkMap::ChunkMap(uint64_t s) - :_file_size(s),_chunk_size(1024*1024) // 1MB chunks +ChunkMap::ChunkMap(uint64_t s,bool assume_availability) + : _file_size(s), + _chunk_size(CHUNKMAP_FIXED_CHUNK_SIZE), // 1MB chunks + _assume_availability(assume_availability) { uint64_t n = s/(uint64_t)_chunk_size ; if(s% (uint64_t)_chunk_size != 0) @@ -62,35 +64,6 @@ void ChunkMap::setAvailabilityMap(const CompressedChunkMap& map) _map[i] = FileChunksInfo::CHUNK_OUTSTANDING ; } -//ChunkMap::ChunkMap(uint64_t file_size, -// const std::vector& map, -// uint32_t chunk_size, -// uint32_t chunk_number, -// FileChunksInfo::ChunkStrategy strategy) -// -// :_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy) -//{ -//#ifdef DEBUG_FTCHUNK -// std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ; -//#endif -// -// _map.clear() ; -// _map.resize(chunk_number) ; -// _total_downloaded = 0 ; -// -// for(uint32_t i=0;i<_map.size();++i) -// { -// uint32_t j = i & 31 ; // i%32 -// uint32_t k = i >> 5 ; // i/32 -// -// _map[i] = ( (map[k] & (1< 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ; -// -// if(_map[i] == FileChunksInfo::CHUNK_DONE) -// _total_downloaded += _chunk_size ; -// } -//} - - void ChunkMap::dataReceived(const ftChunk::ChunkId& cid) { // 1 - find which chunk contains the received data. @@ -271,49 +244,53 @@ uint32_t ChunkMap::getAvailableChunk(uint32_t start_location,const std::string& // Very bold algorithm: checks for 1st availabe chunk for this peer starting // from the given start location. std::map::iterator it(_peers_chunks_availability.find(peer_id)) ; + SourceChunksInfo *peer_chunks = NULL; // Do we have records for this file source ? // - if(it == _peers_chunks_availability.end()) + if(!_assume_availability) { - SourceChunksInfo& pchunks(_peers_chunks_availability[peer_id]) ; + if(it == _peers_chunks_availability.end()) + { + SourceChunksInfo& pchunks(_peers_chunks_availability[peer_id]) ; - // Ok, we don't have the info, so two cases: - // - we are the actual source, so we can safely init the map to a full map - // - we are not the source, so we init it with an empty map, and set the time stamp to 0. + // Ok, we don't have the info, so two cases: + // - we are the actual source, so we can safely init the map to a full map + // - we are not the source, so we init it with an empty map, and set the time stamp to 0. + // + if(peer_id == rsPeers->getOwnId()) + { + pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),~(uint32_t)0 ) ; + pchunks.TS = 0 ; + pchunks.is_full = true ; + } + else + { + pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),0 ) ; + pchunks.TS = 0 ; + pchunks.is_full = false ; + } + + it = _peers_chunks_availability.find(peer_id) ; + } + peer_chunks = &(it->second) ; + + // If the info is too old, we ask for a new one. When the map is full, we ask 10 times less, as it's probably not + // useful to get a new map that will also be full, but because we need to be careful not to mislead information, + // we still keep asking. // - if(peer_id == rsPeers->getOwnId()) - { - pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),~(uint32_t)0 ) ; - pchunks.TS = 0 ; - pchunks.is_full = true ; - } - else - { - pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),0 ) ; - pchunks.TS = 0 ; - pchunks.is_full = false ; - } + time_t now = time(NULL) ; + map_is_too_old = (int)now - (int)peer_chunks->TS > (int)SOURCE_CHUNK_MAP_UPDATE_PERIOD*(1+9*peer_chunks->is_full) ; - it = _peers_chunks_availability.find(peer_id) ; + // We will re-ask but not now seconds. + peer_chunks->TS = now ; } - SourceChunksInfo& peer_chunks(it->second) ; - - // If the info is too old, we ask for a new one. When the map is full, we ask 10 times less, as it's probably not - // useful to get a new map that will also be full, but because we need to be careful not to mislead information, - // we still keep asking. - // - time_t now = time(NULL) ; - map_is_too_old = (int)now - (int)peer_chunks.TS > (int)SOURCE_CHUNK_MAP_UPDATE_PERIOD*(1+9*peer_chunks.is_full) ; - - // We will re-ask but not now seconds. - peer_chunks.TS = now ; for(unsigned int i=0;i<_map.size();++i) { uint32_t j = (start_location+i)%(int)_map.size() ; // index of the chunk - if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && peer_chunks.cmap[j]) + if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && (_assume_availability || peer_chunks->cmap[j])) { #ifdef DEBUG_FTCHUNK std::cerr << "ChunkMap::getAvailableChunk: returning chunk " << j << " for peer " << peer_id << std::endl; @@ -356,7 +333,7 @@ void ChunkMap::getAvailabilityMap(CompressedChunkMap& compressed_map) const void ChunkMap::buildPlainMap(uint64_t size, CompressedChunkMap& map) { - uint32_t chunk_size(1024*1024) ; // 1MB chunks + uint32_t chunk_size(CHUNKMAP_FIXED_CHUNK_SIZE) ; // 1MB chunks uint64_t n = size/(uint64_t)chunk_size ; if(size % (uint64_t)chunk_size != 0) diff --git a/libretroshare/src/ft/ftchunkmap.h b/libretroshare/src/ft/ftchunkmap.h index 314e3b58a..92af90275 100644 --- a/libretroshare/src/ft/ftchunkmap.h +++ b/libretroshare/src/ft/ftchunkmap.h @@ -84,11 +84,12 @@ class SourceChunksInfo class ChunkMap { public: + static const uint32_t CHUNKMAP_FIXED_CHUNK_SIZE = 1024*1024 ; // 1 MB chunk typedef uint32_t ChunkNumber ; /// Constructor. Decides what will be the size of chunks and how many there will be. - ChunkMap(uint64_t file_size) ; + ChunkMap(uint64_t file_size,bool assume_sources_availability) ; /// constructor from saved map info ChunkMap(uint64_t file_size,const std::vector& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ; @@ -163,6 +164,7 @@ class ChunkMap std::vector _map ; //! vector of chunk state over the whole file std::map _peers_chunks_availability ; //! what does each source peer have uint64_t _total_downloaded ; //! completion for the file + bool _assume_availability ; //! set for transfers where the source is always available }; diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index e5233105c..3a805ff28 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -472,6 +472,7 @@ bool ftController::completeFile(std::string hash) mDownloads.erase(it); mTurtle->stopMonitoringFileTunnels(hash_to_suppress) ; + } /******* UNLOCKED ********/ @@ -770,8 +771,7 @@ bool ftController::FileRequest(std::string fname, std::string hash, std::cerr << std::endl; #endif - if (srcIds.end() == std::find( - srcIds.begin(), srcIds.end(), pit->peerId)) + if (srcIds.end() == std::find( srcIds.begin(), srcIds.end(), pit->peerId)) { srcIds.push_back(pit->peerId); @@ -814,7 +814,18 @@ bool ftController::FileRequest(std::string fname, std::string hash, } } /******* UNLOCKED ********/ - ftFileCreator *fc = new ftFileCreator(savepath, size, hash, 0); + // We check that flags are consistent. In particular, for know + // we can't send chunkmaps through normal traffic, so availability must be assumed whenever the traffic is not + // a turtle traffic: + + if(flags & RS_FILE_HINTS_NETWORK_WIDE) + mTurtle->monitorFileTunnels(fname,hash,size) ; + else + flags |= RS_FILE_HINTS_ASSUME_AVAILABILITY ; + + bool assume_source_availability = (flags & RS_FILE_HINTS_ASSUME_AVAILABILITY) > 0 ; + + ftFileCreator *fc = new ftFileCreator(savepath, size, hash, 0,assume_source_availability); ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this); /* add into maps */ @@ -908,6 +919,8 @@ bool ftController::setChunkStrategy(const std::string& hash,FileChunksInfo::Chun bool ftController::FileCancel(std::string hash) { + rsTurtle->stopMonitoringFileTunnels(hash) ; + #ifdef CONTROL_DEBUG std::cerr << "ftController::FileCancel" << std::endl; #endif @@ -1356,7 +1369,7 @@ bool ftController::RequestCacheFile(RsPeerId id, std::string path, std::string h std::list ids; ids.push_back(id); - FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids); + FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH | RS_FILE_HINTS_ASSUME_AVAILABILITY, ids); return true; } diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index 5e2c61505..4bb3d7258 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -15,8 +15,8 @@ * ***********************************************************/ -ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash, uint64_t recvd) - : ftFileProvider(path,size,hash), chunkMap(size) +ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash, uint64_t recvd,bool assume_sources_availability) + : ftFileProvider(path,size,hash), chunkMap(size,assume_sources_availability) { /* * FIXME any inits to do? diff --git a/libretroshare/src/ft/ftfilecreator.h b/libretroshare/src/ft/ftfilecreator.h index 37f48f228..baf4b68c9 100644 --- a/libretroshare/src/ft/ftfilecreator.h +++ b/libretroshare/src/ft/ftfilecreator.h @@ -40,7 +40,7 @@ class ftFileCreator: public ftFileProvider { public: - ftFileCreator(std::string savepath, uint64_t size, std::string hash, uint64_t recvd); + ftFileCreator(std::string savepath, uint64_t size, std::string hash, uint64_t recvd,bool assume_sources_availability=false); ~ftFileCreator(); diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index c660eea92..92675aa63 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -242,17 +242,13 @@ void ftServer::run() /********************** Controller Access **********************/ /***************************************************************/ -bool ftServer::FileRequest(std::string fname, std::string hash, uint64_t size, - std::string dest, uint32_t flags, std::list srcIds) +bool ftServer::FileRequest(std::string fname, std::string hash, uint64_t size, std::string dest, uint32_t flags, std::list srcIds) { std::cerr << "Requesting " << fname << std::endl ; // return mFtController->FileRequest(fname, hash, size, // dest, flags, srcIds); const DwlDetails details(fname, hash, size, dest, flags, srcIds, Normal); mFtDwlQueue->insertDownload(details); - - rsTurtle->monitorFileTunnels( fname,hash,size ) ; - return true ; } @@ -263,8 +259,6 @@ bool ftServer::setChunkStrategy(const std::string& hash,FileChunksInfo::ChunkStr bool ftServer::FileCancel(std::string hash) { - rsTurtle->stopMonitoringFileTunnels(hash) ; - return mFtController->FileCancel(hash); } diff --git a/libretroshare/src/rsiface/rsfiles.h b/libretroshare/src/rsiface/rsfiles.h index c1dcc1820..9fc05af4a 100644 --- a/libretroshare/src/rsiface/rsfiles.h +++ b/libretroshare/src/rsiface/rsfiles.h @@ -60,17 +60,18 @@ const uint32_t RS_FILE_PEER_OFFLINE = 0x00002000; * */ -const uint32_t RS_FILE_HINTS_MASK = 0x00ffffff; +const uint32_t RS_FILE_HINTS_MASK = 0x00ffffff; -const uint32_t RS_FILE_HINTS_CACHE = 0x00000001; -const uint32_t RS_FILE_HINTS_EXTRA = 0x00000002; -const uint32_t RS_FILE_HINTS_LOCAL = 0x00000004; -const uint32_t RS_FILE_HINTS_REMOTE = 0x00000008; -const uint32_t RS_FILE_HINTS_DOWNLOAD = 0x00000010; -const uint32_t RS_FILE_HINTS_UPLOAD = 0x00000020; -//const uint32_t RS_FILE_HINTS_TURTLE = 0x00000040; // unused -const uint32_t RS_FILE_HINTS_NETWORK_WIDE = 0x00000080; // anonymously shared over network -const uint32_t RS_FILE_HINTS_BROWSABLE = 0x00000100; // browsable by friends +const uint32_t RS_FILE_HINTS_CACHE = 0x00000001; +const uint32_t RS_FILE_HINTS_EXTRA = 0x00000002; +const uint32_t RS_FILE_HINTS_LOCAL = 0x00000004; +const uint32_t RS_FILE_HINTS_REMOTE = 0x00000008; +const uint32_t RS_FILE_HINTS_DOWNLOAD = 0x00000010; +const uint32_t RS_FILE_HINTS_UPLOAD = 0x00000020; + +const uint32_t RS_FILE_HINTS_NETWORK_WIDE = 0x00000080; // anonymously shared over network +const uint32_t RS_FILE_HINTS_BROWSABLE = 0x00000100; // browsable by friends +const uint32_t RS_FILE_HINTS_ASSUME_AVAILABILITY = 0x00000200; // Assume full source availability. Used for cache files. const uint32_t RS_FILE_HINTS_SPEC_ONLY = 0x01000000; const uint32_t RS_FILE_HINTS_NO_SEARCH = 0x02000000; diff --git a/retroshare-gui/src/gui/SearchDialog.cpp b/retroshare-gui/src/gui/SearchDialog.cpp index e32f00981..c6f1debe9 100644 --- a/retroshare-gui/src/gui/SearchDialog.cpp +++ b/retroshare-gui/src/gui/SearchDialog.cpp @@ -252,7 +252,7 @@ void SearchDialog::download() rsFiles -> FileRequest((item->text(SR_NAME_COL)).toStdString(), (item->text(SR_HASH_COL)).toStdString(), (item->text(SR_REALSIZE_COL)).toInt(), - "", 0, srcIds); + "", RS_FILE_HINTS_NETWORK_WIDE, srcIds); std::cout << "isuing file request from search dialog: -" << (item->text(SR_NAME_COL)).toStdString() << "-" << (item->text(SR_HASH_COL)).toStdString() << "-" << (item->text(SR_REALSIZE_COL)).toInt() << "-ids=" ; for(std::list::const_iterator it(srcIds.begin());it!=srcIds.end();++it) @@ -286,7 +286,7 @@ void SearchDialog::downloadDirectory(const QTreeWidgetItem *item, const QString rsFiles->FileRequest(item->text(SR_NAME_COL).toStdString(), item->text(SR_HASH_COL).toStdString(), item->text(SR_REALSIZE_COL).toInt(), - cleanPath.toStdString(), 0, srcIds); + cleanPath.toStdString(),RS_FILE_HINTS_NETWORK_WIDE, srcIds); std::cout << "SearchDialog::downloadDirectory(): "\ "issuing file request from search dialog: -"