Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-21 21:55:15 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

- Implemented chunk-based file transfer from partial sources. This in particular means:

Browse Source 
- exchange of chunk availability maps from different peers
    - correct handling of what is available to which source before asking the data
    - correct display of chunks in the progress bars
    - generalised the use of compressed chunk maps
    - removed the size parameters from the hash search functions
   
- In addition:
    - suppressed a number of per-value transfers of std::string
    - improved the FileTransferInfo Widget, to show some additional info

Still to be done:
    - chunk map exchange for non anonymous traffic (easy)
    - improve accuracy of completion for uploads (for now it's a integer number of chunks)
    - check compilation on windows




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1993 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2010-01-11 16:00:42 +00:00
parent add5d45eeb
commit cfaaec31c7
36 changed files with 1247 additions and 573 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

206
libretroshare/src/ft/ftchunkmap.cc
View File

@ -1,8 +1,11 @@
#include <math.h> #include <math.h>
#include <assert.h> #include <assert.h>
#include <stdlib.h> #include <stdlib.h>
#include <rsiface/rspeers.h>
#include "ftchunkmap.h" #include "ftchunkmap.h"
static const uint32_t SOURCE_CHUNK_MAP_UPDATE_PERIOD = 60 ; //! TTL for chunkmap info
std::ostream& operator<<(std::ostream& o,const ftChunk& c) std::ostream& operator<<(std::ostream& o,const ftChunk& c)
{ {
return o << "\tChunk [" << c.offset << "] size: " << c.size << " ChunkId: " << c.id << " Age: " << time(NULL) - c.ts ; return o << "\tChunk [" << c.offset << "] size: " << c.size << " ChunkId: " << c.id << " Age: " << time(NULL) - c.ts ;
@ -28,26 +31,6 @@ void Chunk::getSlice(uint32_t size_hint,ftChunk& chunk)
_offset += chunk.size ; _offset += chunk.size ;
} }
//uint32_t Chunk::dataReceived(const ftChunk::ChunkId cid)
//{
//#ifdef DEBUG_FTCHUNK
// std::cerr << "*** Chunk::dataReceived: slice " << cid << " finished" << std::endl ;
//#endif
// std::map<ftChunk::ChunkId,uint32_t>::iterator it( _slices_to_download.find(cid) ) ;
//
// if(it == _slices_to_download.end())
// {
// std::cerr << "!!! Chunk::dataReceived: could not find chunk " << cid << ": probably a fatal error" << std::endl ;
// return 0 ;
// }
// else
// {
// uint32_t n = it->second ;
// _slices_to_download.erase(it) ;
// return n ;
// }
//}
ChunkMap::ChunkMap(uint64_t s) ChunkMap::ChunkMap(uint64_t s)
:_file_size(s),_chunk_size(1024*1024) // 1MB chunks :_file_size(s),_chunk_size(1024*1024) // 1MB chunks
{ {
@ -67,34 +50,46 @@ ChunkMap::ChunkMap(uint64_t s)
#endif #endif
} }
ChunkMap::ChunkMap(uint64_t file_size, void ChunkMap::setAvailabilityMap(const CompressedChunkMap& map)
const std::vector<uint32_t>& map,
uint32_t chunk_size,
uint32_t chunk_number,
FileChunksInfo::ChunkStrategy strategy)
:_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy)
{ {
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ;
#endif
_map.clear() ;
_map.resize(chunk_number) ;
_total_downloaded = 0 ;
for(uint32_t i=0;i<_map.size();++i) for(uint32_t i=0;i<_map.size();++i)
{ if(map[i] > 0)
uint32_t j = i & 31 ; // i%32 {
uint32_t k = i >> 5 ; // i/32 _map[i] = FileChunksInfo::CHUNK_DONE ;
_map[i] = ( (map[k] & (1<<j)) > 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ;
if(_map[i] == FileChunksInfo::CHUNK_DONE)
_total_downloaded += _chunk_size ; _total_downloaded += _chunk_size ;
} }
else
_map[i] = FileChunksInfo::CHUNK_OUTSTANDING ;
} }
//ChunkMap::ChunkMap(uint64_t file_size,
// const std::vector<uint32_t>& map,
// uint32_t chunk_size,
// uint32_t chunk_number,
// FileChunksInfo::ChunkStrategy strategy)
//
// :_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy)
//{
//#ifdef DEBUG_FTCHUNK
// std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ;
//#endif
//
// _map.clear() ;
// _map.resize(chunk_number) ;
// _total_downloaded = 0 ;
//
// for(uint32_t i=0;i<_map.size();++i)
// {
// uint32_t j = i & 31 ; // i%32
// uint32_t k = i >> 5 ; // i/32
//
// _map[i] = ( (map[k] & (1<<j)) > 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ;
//
// if(_map[i] == FileChunksInfo::CHUNK_DONE)
// _total_downloaded += _chunk_size ;
// }
//}
void ChunkMap::dataReceived(const ftChunk::ChunkId& cid) void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
{ {
@ -151,7 +146,7 @@ void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
// - chunks pushed when new chunks are needed // - chunks pushed when new chunks are needed
// - chunks removed when completely downloaded // - chunks removed when completely downloaded
// //
bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk) bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk,bool& source_chunk_map_needed)
{ {
#ifdef DEBUG_FTCHUNK #ifdef DEBUG_FTCHUNK
std::cerr << "*** ChunkMap::getDataChunk: size_hint = " << size_hint << std::endl ; std::cerr << "*** ChunkMap::getDataChunk: size_hint = " << size_hint << std::endl ;
@ -168,10 +163,10 @@ bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChun
switch(_strategy) switch(_strategy)
{ {
case FileChunksInfo::CHUNK_STRATEGY_STREAMING: c = getAvailableChunk(0,peer_id) ; // very bold!! case FileChunksInfo::CHUNK_STRATEGY_STREAMING: c = getAvailableChunk(0,peer_id,source_chunk_map_needed) ; // very bold!!
break ; break ;
case FileChunksInfo::CHUNK_STRATEGY_RANDOM: c = getAvailableChunk(rand()%_map.size(),peer_id) ; case FileChunksInfo::CHUNK_STRATEGY_RANDOM: c = getAvailableChunk(rand()%_map.size(),peer_id,source_chunk_map_needed) ;
break ; break ;
default: default:
#ifdef DEBUG_FTCHUNK #ifdef DEBUG_FTCHUNK
@ -212,23 +207,51 @@ bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChun
return true ; return true ;
} }
void ChunkMap::setPeerAvailabilityMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& compressed_peer_map) bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const
{
uint32_t chunk_number_start = offset/(uint64_t)_chunk_size ;
uint32_t chunk_number_end = (offset+(uint64_t)chunk_size)/(uint64_t)_chunk_size ;
if((offset+(uint64_t)chunk_size) % (uint64_t)_chunk_size == 0)
--chunk_number_end ;
// It's possible that chunk_number_start==chunk_number_end+1, but for this we need to have
// chunk_size=0, and offset%_chunk_size=0, so the response "true" is still valid.
//
for(uint32_t i=chunk_number_start;i!=chunk_number_end;++i)
if(_map[i] != FileChunksInfo::CHUNK_DONE)
return false ;
return true ;
}
void ChunkMap::setPeerAvailabilityMap(const std::string& peer_id,const CompressedChunkMap& cmap)
{ {
#ifdef DEBUG_FTCHUNK #ifdef DEBUG_FTCHUNK
std::cout << "ChunkMap::Receiving new availability map for peer " << peer_id << std::endl ; std::cout << "ChunkMap::Receiving new availability map for peer " << peer_id << std::endl ;
#endif #endif
// Check that the parameters are the same. Otherwise we should convert the info into the local format.
// If all RS instances have the same policy for deciding the sizes of chunks, this should not happen.
if(chunk_size != _chunk_size || nb_chunks != _map.size()) if(cmap._map.size() != _map.size()/32+(_map.size()%32 != 0))
{ {
std::cerr << "ChunkMap::setPeerAvailabilityMap: chunk size / number of chunks is not correct. Dropping the info." << std::endl ; std::cerr << "ChunkMap::setPeerAvailabilityMap: chunk size / number of chunks is not correct. Dropping the info. cmap.size()=" << cmap._map.size() << ", _map/32+0/1 = " << _map.size()/32+(_map.size()%32 != 0) << std::endl ;
return ; return ;
} }
// sets the map. // sets the map.
// //
_peers_chunks_availability[peer_id] = compressed_peer_map ; SourceChunksInfo& mi(_peers_chunks_availability[peer_id]) ;
mi.cmap = cmap ;
mi.TS = time(NULL) ;
mi.is_full = true ;
// Checks wether the map is full of not.
//
for(uint i=0;i<_map.size();++i)
if(!cmap[i])
{
mi.is_full = false ;
break ;
}
#ifdef DEBUG_FTCHUNK #ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap::setPeerAvailabilityMap: Setting chunk availability info for peer " << peer_id << std::endl ; std::cerr << "ChunkMap::setPeerAvailabilityMap: Setting chunk availability info for peer " << peer_id << std::endl ;
@ -243,32 +266,54 @@ uint32_t ChunkMap::sizeOfChunk(uint32_t cid) const
return _chunk_size ; return _chunk_size ;
} }
uint32_t ChunkMap::getAvailableChunk(uint32_t start_location,const std::string& peer_id) uint32_t ChunkMap::getAvailableChunk(uint32_t start_location,const std::string& peer_id,bool& map_is_too_old)
{ {
// Very bold algorithm: checks for 1st availabe chunk for this peer starting // Very bold algorithm: checks for 1st availabe chunk for this peer starting
// from the given start location. // from the given start location.
std::map<std::string,std::vector<uint32_t> >::const_iterator it(_peers_chunks_availability.find(peer_id)) ; std::map<std::string,SourceChunksInfo>::iterator it(_peers_chunks_availability.find(peer_id)) ;
// Do we have records for this file source ? // Do we have records for this file source ?
// //
if(it == _peers_chunks_availability.end()) if(it == _peers_chunks_availability.end())
{ {
#ifdef DEBUG_FTCHUNK SourceChunksInfo& pchunks(_peers_chunks_availability[peer_id]) ;
std::cout << "No chunk map for peer " << peer_id << ": supposing full data." << std::endl ;
#endif
std::vector<uint32_t>& pchunks(_peers_chunks_availability[peer_id]) ;
pchunks.resize( (_map.size() >> 5)+!!(_map.size() & 0x11111),~(uint32_t)0 ) ; // Ok, we don't have the info, so two cases:
// - we are the actual source, so we can safely init the map to a full map
// - we are not the source, so we init it with an empty map, and set the time stamp to 0.
//
if(peer_id == rsPeers->getOwnId())
{
pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),~(uint32_t)0 ) ;
pchunks.TS = 0 ;
pchunks.is_full = true ;
}
else
{
pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),0 ) ;
pchunks.TS = 0 ;
pchunks.is_full = false ;
}
it = _peers_chunks_availability.find(peer_id) ; it = _peers_chunks_availability.find(peer_id) ;
} }
const std::vector<uint32_t>& peer_chunks(it->second) ; SourceChunksInfo& peer_chunks(it->second) ;
// If the info is too old, we ask for a new one. When the map is full, we ask 10 times less, as it's probably not
// useful to get a new map that will also be full, but because we need to be careful not to mislead information,
// we still keep asking.
//
time_t now = time(NULL) ;
map_is_too_old = (int)now - (int)peer_chunks.TS > (int)SOURCE_CHUNK_MAP_UPDATE_PERIOD*(1+9*peer_chunks.is_full) ;
// We will re-ask but not now seconds.
peer_chunks.TS = now ;
for(unsigned int i=0;i<_map.size();++i) for(unsigned int i=0;i<_map.size();++i)
{ {
uint32_t j = (start_location+i)%(int)_map.size() ; // index of the chunk uint32_t j = (start_location+i)%(int)_map.size() ; // index of the chunk
if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && COMPRESSED_MAP_READ(peer_chunks,j)) if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && peer_chunks.cmap[j])
{ {
#ifdef DEBUG_FTCHUNK #ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap::getAvailableChunk: returning chunk " << j << " for peer " << peer_id << std::endl; std::cerr << "ChunkMap::getAvailableChunk: returning chunk " << j << " for peer " << peer_id << std::endl;
@ -296,35 +341,30 @@ void ChunkMap::getChunksInfo(FileChunksInfo& info) const
info.compressed_peer_availability_maps.clear() ; info.compressed_peer_availability_maps.clear() ;
for(std::map<std::string,std::vector<uint32_t> >::const_iterator it(_peers_chunks_availability.begin());it!= _peers_chunks_availability.end();++it) for(std::map<std::string,SourceChunksInfo>::const_iterator it(_peers_chunks_availability.begin());it!=_peers_chunks_availability.end();++it)
info.compressed_peer_availability_maps.push_back(std::pair<std::string,std::vector<uint32_t> >(it->first,it->second)) ; info.compressed_peer_availability_maps[it->first] = it->second.cmap ;
} }
void ChunkMap::buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& strategy) const void ChunkMap::getAvailabilityMap(CompressedChunkMap& compressed_map) const
{ {
chunk_size = _chunk_size ; compressed_map = CompressedChunkMap(_map) ;
chunk_number = _map.size() ;
strategy = _strategy ;
map.clear() ;
map.reserve((chunk_number >> 5)+1) ;
uint32_t r=0 ;
for(uint32_t i=0;i<_map.size();++i)
{
uint32_t j = i & 31 ;
r |= (_map[i]==FileChunksInfo::CHUNK_DONE)?(1<<j):0 ;
if(j==31 || i==_map.size()-1)
{
map.push_back(r);
r=0 ;
}
}
#ifdef DEBUG_FTCHUNK #ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap:: built availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << ", strategy=" << _strategy << std::endl ; std::cerr << "ChunkMap:: retrieved availability map of size " << _map.size() << ", chunk_size=" << _chunk_size << std::endl ;
#endif #endif
} }
void ChunkMap::buildPlainMap(uint64_t size, CompressedChunkMap& map)
{
uint32_t chunk_size(1024*1024) ; // 1MB chunks
uint64_t n = size/(uint64_t)chunk_size ;
if(size % (uint64_t)chunk_size != 0)
++n ;
map = CompressedChunkMap(n,~uint32_t(0)) ;
}

98
libretroshare/src/ft/ftchunkmap.h
View File

@ -73,80 +73,96 @@ class ChunkDownloadInfo
uint32_t _remains ; uint32_t _remains ;
}; };
class SourceChunksInfo
{
public:
CompressedChunkMap cmap ; //! map of what the peer has/doens't have
time_t TS ; //! last update time for this info
bool is_full ; //! is the map full ? In such a case, re-asking for it is unnecessary.
};
class ChunkMap class ChunkMap
{ {
public: public:
typedef uint32_t ChunkNumber ; typedef uint32_t ChunkNumber ;
// Constructor. Decides what will be the size of chunks and how many there will be. /// Constructor. Decides what will be the size of chunks and how many there will be.
ChunkMap(uint64_t file_size) ; ChunkMap(uint64_t file_size) ;
// constructor from saved map info /// constructor from saved map info
ChunkMap(uint64_t file_size,const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ; ChunkMap(uint64_t file_size,const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
// destructor /// destructor
virtual ~ChunkMap() {} virtual ~ChunkMap() {}
// Returns an slice of data to be asked to the peer within a chunk. /// Returns an slice of data to be asked to the peer within a chunk.
// If a chunk is already been downloaded by this peer, take a slice at /// If a chunk is already been downloaded by this peer, take a slice at
// the beginning of this chunk, or at least where it starts. /// the beginning of this chunk, or at least where it starts.
// If not, randomly/streamly select a new chunk depending on the strategy. /// If not, randomly/streamly select a new chunk depending on the strategy.
// adds an entry in the chunk_ids map, and sets up 1 interval for it. /// adds an entry in the chunk_ids map, and sets up 1 interval for it.
// the chunk should be available from the designated peer. /// the chunk should be available from the designated peer.
virtual bool getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk) ; virtual bool getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk,bool& source_chunk_map_needed) ;
// Notify received a slice of data. This needs to /// Notify received a slice of data. This needs to
// - carve in the map of chunks what is received, what is not. /// - carve in the map of chunks what is received, what is not.
// - tell which chunks are finished. For this, each interval must know what chunk number it has been attributed /// - tell which chunks are finished. For this, each interval must know what chunk number it has been attributed
// when the interval is split in the middle, the number of intervals for the chunk is increased. If the interval is /// when the interval is split in the middle, the number of intervals for the chunk is increased. If the interval is
// completely covered by the data, the interval number is decreased. /// completely covered by the data, the interval number is decreased.
virtual void dataReceived(const ftChunk::ChunkId& c_id) ; virtual void dataReceived(const ftChunk::ChunkId& c_id) ;
// Decides how chunks are selected. /// Decides how chunks are selected.
// STREAMING: the 1st chunk is always returned /// STREAMING: the 1st chunk is always returned
// RANDOM: the beginning of a random interval is selected first. If two few intervals /// RANDOM: the beginning of a random interval is selected first. If two few intervals
// exist, the largest one is randomly split into two. /// exist, the largest one is randomly split into two.
void setStrategy(FileChunksInfo::ChunkStrategy s) { _strategy = s ; } void setStrategy(FileChunksInfo::ChunkStrategy s) { _strategy = s ; }
FileChunksInfo::ChunkStrategy getStrategy() const { return _strategy ; }
// Properly fills an vector of fixed size chunks with availability or download state. /// Properly fills an vector of fixed size chunks with availability or download state.
// chunks is given with the proper number of chunks and we have to adapt to it. This can be used /// chunks is given with the proper number of chunks and we have to adapt to it. This can be used
// to display square chunks in the gui or display a blue bar of availability by collapsing info from all peers. /// to display square chunks in the gui or display a blue bar of availability by collapsing info from all peers.
/// The set method is not virtual because it has no reason to exist in the parent ftFileProvider
void buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& s) const ; virtual void getAvailabilityMap(CompressedChunkMap& cmap) const ;
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ; void setAvailabilityMap(const CompressedChunkMap& cmap) ;
// Updates the peer's availablility map /// This function fills in a plain map for a file of the given size. This
/// is used to ensure that the chunk size will be consistent with the rest
/// of the code.
// //
void setPeerAvailabilityMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& peer_map) ; static void buildPlainMap(uint64_t size,CompressedChunkMap& map) ;
// Returns the total size of downloaded data in the file. /// This function is used by the parent ftFileProvider to know whether the chunk can be sent or not.
bool isChunkAvailable(uint64_t offset, uint32_t chunk_size) const ;
/// Updates the peer's availablility map
//
void setPeerAvailabilityMap(const std::string& peer_id,const CompressedChunkMap& peer_map) ;
/// Returns the total size of downloaded data in the file.
uint64_t getTotalReceived() const { return _total_downloaded ; } uint64_t getTotalReceived() const { return _total_downloaded ; }
void getChunksInfo(FileChunksInfo& info) const ; void getChunksInfo(FileChunksInfo& info) const ;
protected: protected:
// handles what size the last chunk has. /// handles what size the last chunk has.
uint32_t sizeOfChunk(uint32_t chunk_number) const ; uint32_t sizeOfChunk(uint32_t chunk_number) const ;
// Returns the first chunk available starting from start_location for this peer_id. /// Returns the first chunk available starting from start_location for this peer_id.
// //
uint32_t getAvailableChunk(uint32_t start_location,const std::string& peer_id) ; uint32_t getAvailableChunk(uint32_t start_location,const std::string& peer_id,bool& chunk_map_too_old) ;
private: private:
uint64_t _file_size ; // total size of the file in bytes. uint64_t _file_size ; //! total size of the file in bytes.
uint32_t _chunk_size ; // Size of chunks. Common to all chunks. uint32_t _chunk_size ; //! Size of chunks. Common to all chunks.
FileChunksInfo::ChunkStrategy _strategy ; // how do we allocate new chunks FileChunksInfo::ChunkStrategy _strategy ; //! how do we allocate new chunks
std::map<std::string,Chunk> _active_chunks_feed ; // vector of chunks being downloaded. Exactly one chunk per peer id. std::map<std::string,Chunk> _active_chunks_feed ; //! vector of chunks being downloaded. Exactly 1 chunk per peer.
std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; // list of (slice id,slice size) std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; //! list of (slice id,slice size)
std::vector<FileChunksInfo::ChunkState> _map ; //! vector of chunk state over the whole file
std::vector<FileChunksInfo::ChunkState> _map ; // vector of chunk state over the whole file std::map<std::string,SourceChunksInfo> _peers_chunks_availability ; //! what does each source peer have
uint64_t _total_downloaded ; //! completion for the file
std::map<std::string,std::vector<uint32_t> > _peers_chunks_availability ; // what does each source peer have, stored in compressed format.
uint64_t _total_downloaded ;
}; };

53
libretroshare/src/ft/ftcontroller.cc
View File

@ -80,8 +80,12 @@ ftFileControl::ftFileControl(std::string fname,
} }
ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, std::string configDir) ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, std::string configDir)
:CacheTransfer(cs), p3Config(CONFIG_TYPE_FT_CONTROL), mDataplex(dm), mFtActive(false), :CacheTransfer(cs), p3Config(CONFIG_TYPE_FT_CONTROL),
mTurtle(NULL), mShareDownloadDir(true),last_save_time(0) last_save_time(0),
mDataplex(dm),
mTurtle(NULL),
mFtActive(false),
mShareDownloadDir(true)
{ {
/* TODO */ /* TODO */
} }
@ -96,7 +100,7 @@ void ftController::setFtSearchNExtra(ftSearch *search, ftExtraList *list)
mExtraList = list; mExtraList = list;
} }
bool ftController::getFileChunksDetails(const std::string& hash,FileChunksInfo& info) bool ftController::getFileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info)
{ {
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
@ -185,7 +189,7 @@ void ftController::run()
} }
time_t now = time(NULL) ; time_t now = time(NULL) ;
if(now - last_save_time > SAVE_TRANSFERS_DELAY) if((int)now - (int)last_save_time > (int)SAVE_TRANSFERS_DELAY)
{ {
IndicateConfigChanged() ; IndicateConfigChanged() ;
last_save_time = now ; last_save_time = now ;
@ -593,7 +597,10 @@ bool ftController::handleAPendingRequest()
std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ; std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
} }
else else
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ; {
(fit->second).mCreator->setAvailabilityMap(rsft->compressed_chunk_map) ;
(fit->second).mCreator->setChunkStrategy((FileChunksInfo::ChunkStrategy)(rsft->chunk_strategy)) ;
}
delete rsft ; delete rsft ;
mPendingChunkMaps.erase(it) ; mPendingChunkMaps.erase(it) ;
@ -733,10 +740,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
} }
else else
{ {
if (mSearch->search(hash, size, if (mSearch->search(hash, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_SPEC_ONLY, info))
RS_FILE_HINTS_LOCAL |
RS_FILE_HINTS_EXTRA |
RS_FILE_HINTS_SPEC_ONLY, info))
{ {
/* have it already */ /* have it already */
/* add in as completed transfer */ /* add in as completed transfer */
@ -750,10 +754,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
} }
/* do a source search - for any extra sources */ /* do a source search - for any extra sources */
if (mSearch->search(hash, size, if (mSearch->search(hash, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info))
RS_FILE_HINTS_REMOTE |
// RS_FILE_HINTS_TURTLE |
RS_FILE_HINTS_SPEC_ONLY, info))
{ {
/* do something with results */ /* do something with results */
#ifdef CONTROL_DEBUG #ifdef CONTROL_DEBUG
@ -817,8 +818,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this); ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this);
/* add into maps */ /* add into maps */
ftFileControl ftfc(fname, savepath, destination, ftFileControl ftfc(fname, savepath, destination, size, hash, flags, fc, tm, callbackCode);
size, hash, flags, fc, tm, callbackCode);
ftfc.mCreateTime = time(NULL); ftfc.mCreateTime = time(NULL);
#ifdef CONTROL_DEBUG #ifdef CONTROL_DEBUG
@ -1469,7 +1469,23 @@ std::list<RsItem *> ftController::saveList(bool &cleanup)
//rft->flags = fit->second.mFlags; //rft->flags = fit->second.mFlags;
fit->second.mTransfer->getFileSources(rft->allPeerIds.ids); fit->second.mTransfer->getFileSources(rft->allPeerIds.ids);
fit->second.mCreator->storeAvailabilityMap(rft->chunk_map,rft->chunk_size,rft->chunk_number,rft->chunk_strategy) ;
// Remove turtle peers from sources, as they are not supposed to survive a reboot of RS, since they are dynamic sources.
// Otherwize, such sources are unknown from the turtle router, at restart, and never get removed.
//
for(std::list<std::string>::iterator sit(rft->allPeerIds.ids.begin());sit!=rft->allPeerIds.ids.end();)
if(mTurtle->isTurtlePeer(*sit))
{
std::list<std::string>::iterator sittmp(sit) ;
++sittmp ;
rft->allPeerIds.ids.erase(sit) ;
sit = sittmp ;
}
else
++sit ;
fit->second.mCreator->getAvailabilityMap(rft->compressed_chunk_map) ;
rft->chunk_strategy = fit->second.mCreator->getChunkStrategy() ;
saveData.push_back(rft); saveData.push_back(rft);
} }
@ -1528,7 +1544,10 @@ bool ftController::loadList(std::list<RsItem *> load)
continue ; // i.e. don't delete the item! continue ; // i.e. don't delete the item!
} }
else else
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ; {
(fit->second).mCreator->setAvailabilityMap(rsft->compressed_chunk_map) ;
(fit->second).mCreator->setChunkStrategy((FileChunksInfo::ChunkStrategy)(rsft->chunk_strategy)) ;
}
} }
} }

2
libretroshare/src/ft/ftcontroller.h
View File

@ -138,7 +138,7 @@ bool FileCancel(std::string hash);
bool FileControl(std::string hash, uint32_t flags); bool FileControl(std::string hash, uint32_t flags);
bool FileClearCompleted(); bool FileClearCompleted();
bool FlagFileComplete(std::string hash); bool FlagFileComplete(std::string hash);
bool getFileChunksDetails(const std::string& hash,FileChunksInfo& info); bool getFileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info);
/* get Details of File Transfers */ /* get Details of File Transfers */
bool FileDownloads(std::list<std::string> &hashs); bool FileDownloads(std::list<std::string> &hashs);

35
libretroshare/src/ft/ftdata.h
View File

@ -40,19 +40,23 @@
/*************** SEND INTERFACE *******************/ /*************** SEND INTERFACE *******************/
class CompressedChunkMap ;
class ftDataSend class ftDataSend
{ {
public: public:
virtual ~ftDataSend() { return; } virtual ~ftDataSend() { return; }
/* Client Send */ /* Client Send */
virtual bool sendDataRequest(std::string peerId, std::string hash, virtual bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
/* Server Send */ /* Server Send */
virtual bool sendData(std::string peerId, std::string hash, uint64_t size, virtual bool sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data) = 0;
uint64_t offset, uint32_t chunksize, void *data) = 0;
/// Send a request for a chunk map
virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) = 0;
/// Send a chunk map
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) = 0;
}; };
@ -62,18 +66,19 @@ virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
class ftDataRecv class ftDataRecv
{ {
public: public:
virtual ~ftDataRecv() { return; }
virtual ~ftDataRecv() { return; } /* Client Recv */
virtual bool recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data) = 0;
/* Client Recv */ /* Server Recv */
virtual bool recvData(std::string peerId, std::string hash, uint64_t size, virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
uint64_t offset, uint32_t chunksize, void *data) = 0;
/* Server Recv */
virtual bool recvDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
/// Send a request for a chunk map
virtual bool recvChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) = 0;
/// Send a chunk map
virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) = 0;
}; };
/******* Pair of Send/Recv (Only need to handle Send side) ******/ /******* Pair of Send/Recv (Only need to handle Send side) ******/

357
libretroshare/src/ft/ftdatamultiplex.cc
View File

@ -50,8 +50,10 @@ ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
return; return;
} }
const uint32_t FT_DATA = 0x0001; const uint32_t FT_DATA = 0x0001; // data cuhnk to be stored
const uint32_t FT_DATA_REQ = 0x0002; const uint32_t FT_DATA_REQ = 0x0002; // data request to be treated
const uint32_t FT_CLIENT_CHUNK_MAP_REQ = 0x0003; // chunk map request to be treated by client
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map reuqest to be treated by server
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data) ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
:mType(type), mPeerId(peerId), mHash(hash), mSize(size), :mType(type), mPeerId(peerId), mHash(hash), mSize(size),
@ -179,8 +181,7 @@ bool ftDataMultiplex::FileDetails(std::string hash, uint32_t hintsflag, FileI
/*************** SEND INTERFACE (calls ftDataSend) *******************/ /*************** SEND INTERFACE (calls ftDataSend) *******************/
/* Client Send */ /* Client Send */
bool ftDataMultiplex::sendDataRequest(std::string peerId, bool ftDataMultiplex::sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize)
{ {
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::sendDataRequest() Client Send"; std::cerr << "ftDataMultiplex::sendDataRequest() Client Send";
@ -190,9 +191,7 @@ bool ftDataMultiplex::sendDataRequest(std::string peerId,
} }
/* Server Send */ /* Server Send */
bool ftDataMultiplex::sendData(std::string peerId, bool ftDataMultiplex::sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{ {
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::sendData() Server Send"; std::cerr << "ftDataMultiplex::sendData() Server Send";
@ -205,9 +204,7 @@ bool ftDataMultiplex::sendData(std::string peerId,
/*************** RECV INTERFACE (provides ftDataRecv) ****************/ /*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */ /* Client Recv */
bool ftDataMultiplex::recvData(std::string peerId, bool ftDataMultiplex::recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{ {
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvData() Client Recv"; std::cerr << "ftDataMultiplex::recvData() Client Recv";
@ -215,17 +212,14 @@ bool ftDataMultiplex::recvData(std::string peerId,
#endif #endif
/* Store in Queue */ /* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
mRequestQueue.push_back( mRequestQueue.push_back(ftRequest(FT_DATA,peerId,hash,size,offset,chunksize,data));
ftRequest(FT_DATA,peerId,hash,size,offset,chunksize,data));
return true; return true;
} }
/* Server Recv */ /* Server Recv */
bool ftDataMultiplex::recvDataRequest(std::string peerId, bool ftDataMultiplex::recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
{ {
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvDataRequest() Server Recv"; std::cerr << "ftDataMultiplex::recvDataRequest() Server Recv";
@ -239,6 +233,24 @@ bool ftDataMultiplex::recvDataRequest(std::string peerId,
return true; return true;
} }
bool ftDataMultiplex::recvChunkMapRequest(const std::string& peerId, const std::string& hash,bool is_client)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvChunkMapRequest() Server Recv";
std::cerr << std::endl;
#endif
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
if(is_client)
mRequestQueue.push_back(ftRequest(FT_CLIENT_CHUNK_MAP_REQ,peerId,hash,0,0,0,NULL));
else
mRequestQueue.push_back(ftRequest(FT_SERVER_CHUNK_MAP_REQ,peerId,hash,0,0,0,NULL));
return true;
}
/*********** BACKGROUND THREAD OPERATIONS ***********/ /*********** BACKGROUND THREAD OPERATIONS ***********/
bool ftDataMultiplex::workQueued() bool ftDataMultiplex::workQueued()
@ -282,30 +294,44 @@ bool ftDataMultiplex::doWork()
switch(req.mType) switch(req.mType)
{ {
case FT_DATA: case FT_DATA:
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA"; std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
handleRecvData(req.mPeerId, req.mHash, req.mSize, handleRecvData(req.mPeerId, req.mHash, req.mSize, req.mOffset, req.mChunk, req.mData);
req.mOffset, req.mChunk, req.mData); break;
break;
case FT_DATA_REQ: case FT_DATA_REQ:
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA_REQ"; std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA_REQ";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
handleRecvDataRequest(req.mPeerId, req.mHash, handleRecvDataRequest(req.mPeerId, req.mHash, req.mSize, req.mOffset, req.mChunk);
req.mSize, req.mOffset, req.mChunk); break;
break;
default: case FT_CLIENT_CHUNK_MAP_REQ:
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN"; std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CHUNK_MAP_REQ";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
break; handleRecvClientChunkMapRequest(req.mPeerId,req.mHash) ;
break ;
case FT_SERVER_CHUNK_MAP_REQ:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CHUNK_MAP_REQ";
std::cerr << std::endl;
#endif
handleRecvServerChunkMapRequest(req.mPeerId,req.mHash) ;
break ;
default:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN";
std::cerr << std::endl;
#endif
break;
} }
} }
@ -330,40 +356,137 @@ bool ftDataMultiplex::doWork()
std::cerr << "ftDataMultiplex::doWork() Handling Search Request"; std::cerr << "ftDataMultiplex::doWork() Handling Search Request";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
handleSearchRequest(req.mPeerId, req.mHash, req.mSize, if(handleSearchRequest(req.mPeerId, req.mHash))
req.mOffset, req.mChunk); handleRecvDataRequest(req.mPeerId, req.mHash, req.mSize, req.mOffset, req.mChunk) ;
return true; return true;
} }
bool ftDataMultiplex::recvFileMap(const std::string& peerId, const std::string& hash, uint32_t chunk_size, uint32_t nb_chunks, const std::vector<uint32_t>& compressed_map) // A chunk map has arrived. It can be two different situations:
// - an uploader has sent his chunk map, so we need to store it in the corresponding ftFileProvider
// - a source for a download has sent his chunk map, so we need to send it to the corresponding ftFileCreator.
//
bool ftDataMultiplex::recvChunkMap(const std::string& peerId, const std::string& hash,const CompressedChunkMap& compressed_map,bool client)
{ {
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string, ftClient>::iterator it;
if (mClients.end() == (it = mClients.find(hash))) if(client) // is the chunk map for a client, or for a server ?
{ {
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
if(it == mClients.end())
{
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvMap() ERROR: No matching Client!"; std::cerr << "ftDataMultiplex::recvChunkMap() ERROR: No matching Client for hash " << hash << " !";
std::cerr << std::endl;
#endif
/* error */
return false;
}
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvChunkMap() Passing map of file " << hash << ", to FT Module";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
/* error */
return false; (it->second).mCreator->setSourceMap(peerId, compressed_map);
return true ;
}
else
{
std::map<std::string, ftFileProvider *>::iterator it = mServers.find(hash) ;
if(it == mServers.end())
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMap() ERROR: No matching file Provider for hash " << hash ;
std::cerr << std::endl;
#endif
}
it->second->setClientMap(peerId, compressed_map);
return true ;
} }
#ifdef MPLEX_DEBUG return false;
std::cerr << "ftDataMultiplex::handleRecvMap() Passing map to FT Module";
std::cerr << std::endl;
#endif
(it->second).mCreator->setSourceMap(peerId, chunk_size, nb_chunks,compressed_map);
return true;
} }
bool ftDataMultiplex::handleRecvData(std::string peerId, bool ftDataMultiplex::handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash)
std::string hash, uint64_t size, {
CompressedChunkMap cmap ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
if(it == mClients.end())
{
// If we can't find the client, it's not a problem. Chunk maps from
// clients are not essential, as they are only used for display.
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvServerChunkMapRequest() ERROR: No matching Client for hash " << hash ;
std::cerr << ". Performing local search." << std::endl;
#endif
return false;
}
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvServerChunkMapRequest() Sending map of file " << hash << ", to peer " << peerId << std::endl;
#endif
(it->second).mCreator->getAvailabilityMap(cmap);
}
mDataSend->sendChunkMap(peerId,hash,cmap);
return true ;
}
bool ftDataMultiplex::handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash)
{
CompressedChunkMap cmap ;
std::map<std::string, ftFileProvider *>::iterator it ;
bool found = true ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end())
found = false ;
}
if(!found)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() ERROR: No matching file Provider for hash " << hash ;
std::cerr << std::endl;
#endif
if(!handleSearchRequest(peerId,hash))
return false ;
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() A new file Provider has been made up for hash " << hash ;
std::cerr << std::endl;
#endif
}
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
it->second->getAvailabilityMap(cmap);
}
mDataSend->sendChunkMap(peerId,hash,cmap);
return true;
}
bool ftDataMultiplex::handleRecvData(const std::string& peerId,
const std::string& hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data) uint64_t offset, uint32_t chunksize, void *data)
{ {
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
@ -390,9 +513,7 @@ bool ftDataMultiplex::handleRecvData(std::string peerId,
/* called by ftTransferModule */ /* called by ftTransferModule */
bool ftDataMultiplex::handleRecvDataRequest(std::string peerId, bool ftDataMultiplex::handleRecvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
{ {
/**** Find Files *****/ /**** Find Files *****/
@ -412,8 +533,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
std::cerr << "ftDataMultiplex::handleRecvData() Matched to a Client."; std::cerr << "ftDataMultiplex::handleRecvData() Matched to a Client.";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
locked_handleServerRequest((cit->second).mCreator, locked_handleServerRequest((cit->second).mCreator, peerId, hash, size, offset, chunksize);
peerId, hash, size, offset, chunksize);
return true; return true;
} }
@ -424,8 +544,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
std::cerr << "ftDataMultiplex::handleRecvData() Matched to a Provider."; std::cerr << "ftDataMultiplex::handleRecvData() Matched to a Provider.";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
locked_handleServerRequest(sit->second, locked_handleServerRequest(sit->second, peerId, hash, size, offset, chunksize);
peerId, hash, size, offset, chunksize);
return true; return true;
} }
@ -435,9 +554,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
#endif #endif
/* Add to Search Queue */ /* Add to Search Queue */
mSearchQueue.push_back( mSearchQueue.push_back( ftRequest(FT_DATA_REQ, peerId, hash, size, offset, chunksize, NULL));
ftRequest(FT_DATA_REQ, peerId, hash,
size, offset, chunksize, NULL));
return true; return true;
} }
@ -481,6 +598,33 @@ bool ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider,
return false; return false;
} }
bool ftDataMultiplex::getClientChunkMap(const std::string& upload_hash,const std::string& peerId,CompressedChunkMap& cmap)
{
bool too_old ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string,ftFileProvider *>::iterator sit = mServers.find(upload_hash);
if(mServers.end() == sit)
return false ;
sit->second->getClientMap(peerId,cmap,too_old) ;
}
// If the map is too old then we should ask an other map to the peer.
//
if(too_old)
sendChunkMapRequest(peerId,upload_hash);
return true ;
}
bool ftDataMultiplex::sendChunkMapRequest(const std::string& peer_id,const std::string& hash)
{
return mDataSend->sendChunkMapRequest(peer_id,hash);
}
void ftDataMultiplex::deleteServers(const std::list<std::string>& serv) void ftDataMultiplex::deleteServers(const std::list<std::string>& serv)
{ {
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
@ -491,42 +635,42 @@ void ftDataMultiplex::deleteServers(const std::list<std::string>& serv)
if(mServers.end() != sit) if(mServers.end() != sit)
{ {
delete sit->second; // Only delete servers that are not also file creators!
//
if(dynamic_cast<ftFileCreator*>(sit->second) == NULL)
delete sit->second;
mServers.erase(sit); mServers.erase(sit);
} }
} }
} }
bool ftDataMultiplex::handleSearchRequest(std::string peerId, bool ftDataMultiplex::handleSearchRequest(const std::string& peerId, const std::string& hash)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
{ {
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleSearchRequest("; std::cerr << "ftDataMultiplex::handleSearchRequest(";
std::cerr << peerId << ", " << hash << ", " << size << "...)"; std::cerr << peerId << ", " << hash << "...)";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
{ // {
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ // RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
//
/* Check for bad requests */ // /* Check for bad requests */
std::map<std::string, time_t>::iterator bit; // std::map<std::string, time_t>::iterator bit;
if (mUnknownHashs.end() != (bit = mUnknownHashs.find(hash))) // if (mUnknownHashs.end() != (bit = mUnknownHashs.find(hash)))
{ // {
//
#ifdef MPLEX_DEBUG //#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleSearchRequest("; // std::cerr << "ftDataMultiplex::handleSearchRequest(";
std::cerr << " Found Ignore Hash ... done"; // std::cerr << " Found Ignore Hash ... done";
std::cerr << std::endl; // std::cerr << std::endl;
#endif //#endif
//
/* We've previously rejected this one, so ignore */ // /* We've previously rejected this one, so ignore */
return false; // return false;
} // }
} // }
/* /*
@ -535,14 +679,10 @@ bool ftDataMultiplex::handleSearchRequest(std::string peerId,
* (anywhere but remote really) * (anywhere but remote really)
*/ */
FileInfo info; FileInfo info;
uint32_t hintflags = (RS_FILE_HINTS_CACHE | uint32_t hintflags = (RS_FILE_HINTS_CACHE | RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY);
RS_FILE_HINTS_EXTRA |
RS_FILE_HINTS_LOCAL |
RS_FILE_HINTS_SPEC_ONLY);
if (mSearch->search(hash, size, hintflags, info)) if (mSearch->search(hash, hintflags, info))
{ {
#ifdef MPLEX_DEBUG #ifdef MPLEX_DEBUG
@ -551,28 +691,35 @@ bool ftDataMultiplex::handleSearchRequest(std::string peerId,
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
/* setup a new provider */ /* setup a new provider */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
ftFileProvider *provider = ftFileProvider *provider = new ftFileProvider(info.path, info.size, hash);
new ftFileProvider(info.path, size, hash);
mServers[hash] = provider; mServers[hash] = provider;
/* handle request finally */
locked_handleServerRequest(provider,
peerId, hash, size, offset, chunksize);
/* now we should should check if any further requests for the same
* file exists ... (can happen with caches!)
*
* but easier to check pre-search....
*/
return true; return true;
} }
// Now check wether the required file is actually being downloaded. In such a case,
// setup the file provider to be the file creator itself. Warning: this server should not
// be deleted when not used anymore. We need to restrict this to client peers that are
// not ourself, since the file transfer also handles the local cache traffic (this
// is something to be changed soon!!)
//
if(peerId != mOwnId)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string,ftClient>::const_iterator it(mClients.find(hash)) ;
if(it != mClients.end())
{
mServers[hash] = it->second.mCreator ;
return true;
}
}
return false; return false;
} }

32
libretroshare/src/ft/ftdatamultiplex.h
View File

@ -102,23 +102,31 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
/*************** SEND INTERFACE (calls ftDataSend) *******************/ /*************** SEND INTERFACE (calls ftDataSend) *******************/
/* Client Send */ /* Client Send */
bool sendDataRequest(std::string peerId, std::string hash, uint64_t size, bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
uint64_t offset, uint32_t chunksize);
/* Server Send */ /* Server Send */
bool sendData(std::string peerId, std::string hash, uint64_t size, bool sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
uint64_t offset, uint32_t chunksize, void *data);
/* Server/client Send */
bool sendChunkMapRequest(const std::string& peerId, const std::string& hash) ;
/*************** RECV INTERFACE (provides ftDataRecv) ****************/ /*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */ /* Client Recv */
virtual bool recvData(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data); virtual bool recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
virtual bool recvFileMap(const std::string& peerId, const std::string& hash, uint32_t chunk_size, uint32_t nb_chunks, const std::vector<uint32_t>& compressed_map) ;
/// Receive a request for a chunk map
virtual bool recvChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) ;
/// Receive a chunk map
virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ;
/* Server Recv */ /* Server Recv */
virtual bool recvDataRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize); virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
// Returns the chunk map from the file uploading client. Also initiates a chunk map request if this
// map is too old. This supposes that the caller will ask again in a few seconds.
//
bool getClientChunkMap(const std::string& upload_hash,const std::string& peer_id,CompressedChunkMap& map) ;
protected: protected:
@ -129,9 +137,11 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
private: private:
/* Handling Job Queues */ /* Handling Job Queues */
bool handleRecvData(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data); bool handleRecvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
bool handleRecvDataRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize); bool handleRecvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
bool handleSearchRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize); bool handleSearchRequest(const std::string& peerId, const std::string& hash);
bool handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash) ;
/* We end up doing the actual server job here */ /* We end up doing the actual server job here */
bool locked_handleServerRequest(ftFileProvider *provider, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize); bool locked_handleServerRequest(ftFileProvider *provider, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
@ -143,7 +153,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
std::list<ftRequest> mRequestQueue; std::list<ftRequest> mRequestQueue;
std::list<ftRequest> mSearchQueue; std::list<ftRequest> mSearchQueue;
std::map<std::string, time_t> mUnknownHashs; // std::map<std::string, time_t> mUnknownHashs;
ftDataSend *mDataSend; ftDataSend *mDataSend;
ftSearch *mSearch; ftSearch *mSearch;

16
libretroshare/src/ft/ftdbase.cc
View File

@ -37,7 +37,7 @@ ftFiStore::ftFiStore(CacheStrapper *cs, CacheTransfer *cft, NotifyBase *cb_in,
return; return;
} }
bool ftFiStore::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const bool ftFiStore::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{ {
/* could use hintflags to specify which bits of fileinfo to use additionally. /* could use hintflags to specify which bits of fileinfo to use additionally.
eg. hintflags & FT_SEARCH_PEER_ID, then only return matching peers + hash. eg. hintflags & FT_SEARCH_PEER_ID, then only return matching peers + hash.
@ -47,7 +47,7 @@ bool ftFiStore::search(std::string hash, uint64_t size, uint32_t hintflags, File
*/ */
#ifdef DB_DEBUG #ifdef DB_DEBUG
std::cerr << "ftFiStore::search(" << hash << "," << size << "," << hintflags; std::cerr << "ftFiStore::search(" << hash << "," << hintflags;
std::cerr << ")"; std::cerr << ")";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
@ -68,8 +68,8 @@ bool ftFiStore::search(std::string hash, uint64_t size, uint32_t hintflags, File
#endif #endif
bool fullmatch = true; bool fullmatch = true;
if (it->size != size) // if (it->size != size)
fullmatch = false; // fullmatch = false;
#if 0 #if 0
@ -125,13 +125,13 @@ ftFiMonitor::ftFiMonitor(CacheStrapper *cs,NotifyBase *cb_in, std::string cached
return; return;
} }
bool ftFiMonitor::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const bool ftFiMonitor::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{ {
uint64_t fsize; uint64_t fsize;
std::string path; std::string path;
#ifdef DB_DEBUG #ifdef DB_DEBUG
std::cerr << "ftFiMonitor::search(" << hash << "," << size << "," << hintflags; std::cerr << "ftFiMonitor::search(" << hash << "," << hintflags;
std::cerr << ")"; std::cerr << ")";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
@ -269,10 +269,10 @@ ftCacheStrapper::ftCacheStrapper(p3AuthMgr *am, p3ConnectMgr *cm)
} }
/* overloaded search function */ /* overloaded search function */
bool ftCacheStrapper::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const bool ftCacheStrapper::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{ {
#ifdef DB_DEBUG #ifdef DB_DEBUG
std::cerr << "ftCacheStrapper::search(" << hash << "," << size << "," << hintflags; std::cerr << "ftCacheStrapper::search(" << hash << "," << hintflags;
std::cerr << ")"; std::cerr << ")";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif

6
libretroshare/src/ft/ftdbase.h
View File

@ -48,7 +48,7 @@ class ftFiStore: public FileIndexStore, public ftSearch
RsPeerId ownid, std::string cachedir); RsPeerId ownid, std::string cachedir);
/* overloaded search function */ /* overloaded search function */
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
}; };
class ftFiMonitor: public FileIndexMonitor, public ftSearch, public p3Config class ftFiMonitor: public FileIndexMonitor, public ftSearch, public p3Config
@ -57,7 +57,7 @@ class ftFiMonitor: public FileIndexMonitor, public ftSearch, public p3Config
ftFiMonitor(CacheStrapper *cs,NotifyBase *cb_in, std::string cachedir, std::string pid); ftFiMonitor(CacheStrapper *cs,NotifyBase *cb_in, std::string cachedir, std::string pid);
/* overloaded search function */ /* overloaded search function */
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
/* overloaded set dirs enables config indication */ /* overloaded set dirs enables config indication */
virtual void setSharedDirectories(std::list<SharedDirInfo> dirList); virtual void setSharedDirectories(std::list<SharedDirInfo> dirList);
@ -81,7 +81,7 @@ class ftCacheStrapper: public CacheStrapper, public ftSearch
ftCacheStrapper(p3AuthMgr *am, p3ConnectMgr *cm); ftCacheStrapper(p3AuthMgr *am, p3ConnectMgr *cm);
/* overloaded search function */ /* overloaded search function */
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
}; };

4
libretroshare/src/ft/ftextralist.cc
View File

@ -318,14 +318,14 @@ bool ftExtraList::hashExtraFileDone(std::string path, FileInfo &info)
} }
hash = it->second; hash = it->second;
} }
return search(hash, 0, 0, info); return search(hash, 0, info);
} }
/*** /***
* Search Function - used by File Transfer * Search Function - used by File Transfer
* *
**/ **/
bool ftExtraList::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const bool ftExtraList::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{ {
#ifdef DEBUG_ELIST #ifdef DEBUG_ELIST

2
libretroshare/src/ft/ftextralist.h
View File

@ -138,7 +138,7 @@ bool hashExtraFileDone(std::string path, FileInfo &info);
* implementation of ftSearch. * implementation of ftSearch.
* *
**/ **/
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
/*** /***
* Thread Main Loop * Thread Main Loop

84
libretroshare/src/ft/ftfilecreator.cc
View File

@ -38,16 +38,19 @@ ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash,
bool ftFileCreator::getFileData(uint64_t offset, uint32_t &chunk_size, void *data) bool ftFileCreator::getFileData(uint64_t offset, uint32_t &chunk_size, void *data)
{ {
// Only send the data if we actually have it.
//
bool have_it = false ;
{ {
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
if (offset + chunk_size > mStart)
{ have_it = chunkMap.isChunkAvailable(offset, chunk_size) ;
/* don't have the data */
return false;
}
} }
return ftFileProvider::getFileData(offset, chunk_size, data); if(have_it)
return ftFileProvider::getFileData(offset, chunk_size, data);
else
return false ;
} }
uint64_t ftFileCreator::getRecvd() uint64_t ftFileCreator::getRecvd()
@ -225,7 +228,6 @@ int ftFileCreator::locked_notifyReceived(uint64_t offset, uint32_t chunk_size)
/* find the chunk */ /* find the chunk */
std::map<uint64_t, ftChunk>::iterator it = mChunks.find(offset); std::map<uint64_t, ftChunk>::iterator it = mChunks.find(offset);
// bool isFirst = false;
if (it == mChunks.end()) if (it == mChunks.end())
{ {
#ifdef FILE_DEBUG #ifdef FILE_DEBUG
@ -238,11 +240,6 @@ int ftFileCreator::locked_notifyReceived(uint64_t offset, uint32_t chunk_size)
return 0; /* ignoring */ return 0; /* ignoring */
} }
// if (it == mChunks.begin())
// {
// isFirst = true;
// }
ftChunk chunk = it->second; ftChunk chunk = it->second;
mChunks.erase(it); mChunks.erase(it);
@ -256,25 +253,18 @@ int ftFileCreator::locked_notifyReceived(uint64_t offset, uint32_t chunk_size)
else // notify the chunkmap that the slice is finished else // notify the chunkmap that the slice is finished
chunkMap.dataReceived(chunk.id) ; chunkMap.dataReceived(chunk.id) ;
// /* update how much has been completed */
// if (isFirst)
// {
// mStart = offset + chunk_size;
// }
// update chunk map
// if (mChunks.size() == 0)
// {
// mStart = mEnd;
// }
/* otherwise there is another earlier block to go /* otherwise there is another earlier block to go
*/ */
return 1; return 1;
} }
FileChunksInfo::ChunkStrategy ftFileCreator::getChunkStrategy()
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
return chunkMap.getStrategy() ;
}
void ftFileCreator::setChunkStrategy(FileChunksInfo::ChunkStrategy s) void ftFileCreator::setChunkStrategy(FileChunksInfo::ChunkStrategy s)
{ {
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
@ -289,7 +279,7 @@ void ftFileCreator::setChunkStrategy(FileChunksInfo::ChunkStrategy s)
* But can return size = 0, if we are still waiting for the data. * But can return size = 0, if we are still waiting for the data.
*/ */
bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t &offset, uint32_t& size) bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t &offset, uint32_t& size,bool& source_chunk_map_needed)
{ {
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
#ifdef FILE_DEBUG #ifdef FILE_DEBUG
@ -298,17 +288,7 @@ bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hin
std::cerr << std::endl; std::cerr << std::endl;
locked_printChunkMap(); locked_printChunkMap();
#endif #endif
source_chunk_map_needed = false ;
/* check start point */
// if(mStart == mSize)
// {
//#ifdef FILE_DEBUG
// std::cerr << "ffc::getMissingChunk() File Done";
// std::cerr << std::endl;
//#endif
// return false;
// }
/* check for freed chunks */ /* check for freed chunks */
time_t ts = time(NULL); time_t ts = time(NULL);
@ -338,30 +318,17 @@ bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hin
ftChunk chunk ; ftChunk chunk ;
if(!chunkMap.getDataChunk(peer_id,size_hint,chunk)) if(!chunkMap.getDataChunk(peer_id,size_hint,chunk,source_chunk_map_needed))
return false ; return false ;
// if (mSize - mEnd < chunk)
// chunk = mSize - mEnd;
//
// offset = mEnd;
// mEnd += chunk;
// if (chunk > 0)
// {
#ifdef FILE_DEBUG #ifdef FILE_DEBUG
std::cerr << "ffc::getMissingChunk() Retrieved new chunk: " << chunk << std::endl ; std::cerr << "ffc::getMissingChunk() Retrieved new chunk: " << chunk << std::endl ;
// std::cerr << std::endl;
// std::cerr << " mStart: " << mStart << " mEnd: " << mEnd;
// std::cerr << "mSize: " << mSize;
// std::cerr << std::endl;
#endif #endif
mChunks[chunk.offset] = chunk ; mChunks[chunk.offset] = chunk ;
offset = chunk.offset ; offset = chunk.offset ;
size = chunk.size ; size = chunk.size ;
// }
return true; /* cos more data to get */ return true; /* cos more data to get */
} }
@ -382,7 +349,6 @@ bool ftFileCreator::locked_printChunkMap()
#endif #endif
/* check start point */ /* check start point */
// std::cerr << "Size: " << mSize << " Start: " << mStart << " End: " << mEnd;
std::cerr << "\tOutstanding Chunks:"; std::cerr << "\tOutstanding Chunks:";
std::cerr << std::endl; std::cerr << std::endl;
@ -394,23 +360,21 @@ bool ftFileCreator::locked_printChunkMap()
return true; return true;
} }
void ftFileCreator::loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t strategy) void ftFileCreator::setAvailabilityMap(const CompressedChunkMap& cmap)
{ {
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
chunkMap = ChunkMap(mSize,map,chunk_size,chunk_number,FileChunksInfo::ChunkStrategy(strategy)) ; chunkMap.setAvailabilityMap(cmap) ;
} }
void ftFileCreator::storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& strategy) void ftFileCreator::getAvailabilityMap(CompressedChunkMap& map)
{ {
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
FileChunksInfo::ChunkStrategy strat ; chunkMap.getAvailabilityMap(map) ;
chunkMap.buildAvailabilityMap(map,chunk_size,chunk_number,strat) ;
strategy = (uint32_t)strat ;
} }
void ftFileCreator::setSourceMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& compressed_map) void ftFileCreator::setSourceMap(const std::string& peer_id,const CompressedChunkMap& compressed_map)
{ {
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
@ -421,7 +385,7 @@ void ftFileCreator::setSourceMap(const std::string& peer_id,uint32_t chunk_size,
// - then asking the chunkmap which chunks are being downloaded, but actually shouldn't // - then asking the chunkmap which chunks are being downloaded, but actually shouldn't
// - cancelling them in the ftFileCreator, so that they can be re-asked later to another peer. // - cancelling them in the ftFileCreator, so that they can be re-asked later to another peer.
// //
chunkMap.setPeerAvailabilityMap(peer_id,chunk_size,nb_chunks,compressed_map) ; chunkMap.setPeerAvailabilityMap(peer_id,compressed_map) ;
} }

14
libretroshare/src/ft/ftfilecreator.h
View File

@ -50,16 +50,20 @@ class ftFileCreator: public ftFileProvider
uint64_t getRecvd(); uint64_t getRecvd();
void getChunkMap(FileChunksInfo& info) ; void getChunkMap(FileChunksInfo& info) ;
void setChunkStrategy(FileChunksInfo::ChunkStrategy s) ; void setChunkStrategy(FileChunksInfo::ChunkStrategy s) ;
FileChunksInfo::ChunkStrategy getChunkStrategy() ;
/* /*
* creation functions for FileCreator * creation functions for FileCreator
*/ */
// Gets a new variable-sized chunk of size "size_hint" from the given peer id. The returned size, "size" is // Gets a new variable-sized chunk of size "size_hint" from the given peer id. The returned size, "size" is
// at most equal to size_hint. // at most equal to size_hint. chunk_map_needed is set if
// - no chunkmap info is available. In such a case, the chunk info is irrelevant and false is returned.
// - the chunk info is too old. In tis case, true is returned, and the chunks info can be used.
// //
bool getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size); bool getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size,bool& is_chunk_map_too_old);
// actually store data in the file, and update chunks info // actually store data in the file, and update chunks info
// //
@ -71,12 +75,12 @@ class ftFileCreator: public ftFileProvider
// - getting info about current chunks for the GUI // - getting info about current chunks for the GUI
// - sending availability info to the peers for which we also are a source // - sending availability info to the peers for which we also are a source
// //
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t chunk_strategy) ; virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
void storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& chunk_strategy) ; void setAvailabilityMap(const CompressedChunkMap& cmap) ;
// This is called when receiving the availability map from a source peer, for the file being handled. // This is called when receiving the availability map from a source peer, for the file being handled.
// //
void setSourceMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& map) ; void setSourceMap(const std::string& peer_id,const CompressedChunkMap& map) ;
protected: protected:

43
libretroshare/src/ft/ftfileprovider.cc
View File

@ -1,9 +1,12 @@
#include "ftfileprovider.h" #include "ftfileprovider.h"
#include "ftchunkmap.h"
#include "util/rsdir.h" #include "util/rsdir.h"
#include <stdlib.h> #include <stdlib.h>
#include <stdio.h> #include <stdio.h>
static const time_t UPLOAD_CHUNK_MAPS_TIME = 30 ; // time to ask for a new chunkmap from uploaders in seconds.
ftFileProvider::ftFileProvider(std::string path, uint64_t size, std::string ftFileProvider::ftFileProvider(std::string path, uint64_t size, std::string
hash) : mSize(size), hash(hash), file_name(path), fd(NULL),transfer_rate(0),total_size(0) hash) : mSize(size), hash(hash), file_name(path), fd(NULL),transfer_rate(0),total_size(0)
{ {
@ -73,6 +76,13 @@ bool ftFileProvider::FileDetails(FileInfo &info)
return true; return true;
} }
void ftFileProvider::getAvailabilityMap(CompressedChunkMap& cmap)
{
// We are here because the file we deal with is complete. So we return a plain map.
//
ChunkMap::buildPlainMap(mSize,cmap) ;
}
bool ftFileProvider::getFileData(uint64_t offset, uint32_t &chunk_size, void *data) bool ftFileProvider::getFileData(uint64_t offset, uint32_t &chunk_size, void *data)
{ {
@ -168,6 +178,39 @@ bool ftFileProvider::getFileData(uint64_t offset, uint32_t &chunk_size, void *da
return 1; return 1;
} }
void ftFileProvider::setClientMap(const std::string& peer_id,const CompressedChunkMap& cmap)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
std::pair<CompressedChunkMap,time_t>& map_info(clients_chunk_maps[peer_id]) ;
map_info.first = cmap ;
map_info.second = time(NULL) ;
}
void ftFileProvider::getClientMap(const std::string& peer_id,CompressedChunkMap& cmap,bool& map_is_too_old)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
std::map<std::string,std::pair<CompressedChunkMap,time_t> >::iterator it(clients_chunk_maps.find(peer_id)) ;
if(it == clients_chunk_maps.end())
{
clients_chunk_maps[peer_id] = std::pair<CompressedChunkMap,time_t>(CompressedChunkMap(),0) ;
it = clients_chunk_maps.find(peer_id) ;
}
if(time(NULL) - it->second.second > UPLOAD_CHUNK_MAPS_TIME)
{
map_is_too_old = true ;
it->second.second = time(NULL) ; // to avoid re-asking before the TTL
}
else
map_is_too_old = false ;
cmap = it->second.first ;
}
int ftFileProvider::initializeFileAttrs() int ftFileProvider::initializeFileAttrs()
{ {
std::cerr << "ftFileProvider::initializeFileAttrs() Filename: "; std::cerr << "ftFileProvider::initializeFileAttrs() Filename: ";

74
libretroshare/src/ft/ftfileprovider.h
View File

@ -37,45 +37,57 @@
class ftFileProvider class ftFileProvider
{ {
public: public:
ftFileProvider(std::string path, uint64_t size, std::string hash); ftFileProvider(std::string path, uint64_t size, std::string hash);
virtual ~ftFileProvider(); virtual ~ftFileProvider();
virtual bool getFileData(uint64_t offset, uint32_t &chunk_size, void *data); virtual bool getFileData(uint64_t offset, uint32_t &chunk_size, void *data);
virtual bool FileDetails(FileInfo &info); virtual bool FileDetails(FileInfo &info);
std::string getHash(); std::string getHash();
uint64_t getFileSize(); uint64_t getFileSize();
bool fileOk(); bool fileOk();
void setPeerId(const std::string& id) ; void setPeerId(const std::string& id) ;
// Provides a client for the map of chunks actually present in the file. If the provider is also
// a file creator, because the file is actually being downloaded, then the map may be partially complete.
// Otherwize, a plain map is returned.
//
virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
protected: // a ftFileProvider feeds a distant peer. To display what the peers already has, we need to store/read this info.
virtual int initializeFileAttrs(); /* does for both */ void getClientMap(const std::string& peer_id,CompressedChunkMap& cmap,bool& map_is_too_old) ;
void setClientMap(const std::string& peer_id,const CompressedChunkMap& cmap) ;
uint64_t mSize; protected:
std::string hash; virtual int initializeFileAttrs(); /* does for both */
std::string file_name;
FILE *fd;
/* uint64_t mSize;
* Structure to gather statistics FIXME: lastRequestor - figure out a std::string hash;
* way to get last requestor (peerID) std::string file_name;
*/ FILE *fd;
std::string lastRequestor;
uint64_t req_loc;
uint32_t req_size;
time_t lastTS; // used for checking if it's alive
time_t lastTS_t; // used for estimating transfer rate.
// these two are used for speed estimation /*
float transfer_rate ; * Structure to gather statistics FIXME: lastRequestor - figure out a
uint32_t total_size ; * way to get last requestor (peerID)
*/
std::string lastRequestor;
uint64_t req_loc;
uint32_t req_size;
time_t lastTS; // used for checking if it's alive
time_t lastTS_t; // used for estimating transfer rate.
/* // these two are used for speed estimation
* Mutex Required for stuff below float transfer_rate ;
*/ uint32_t total_size ;
RsMutex ftcMutex;
// Info about what the downloading peer already has
std::map<std::string,std::pair<CompressedChunkMap,time_t> > clients_chunk_maps ;
/*
* Mutex Required for stuff below
*/
RsMutex ftcMutex;
}; };

12
libretroshare/src/ft/ftfilesearch.cc
View File

@ -75,12 +75,12 @@ bool ftFileSearch::addSearchMode(ftSearch *search, uint32_t hintflags)
return false; return false;
} }
bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const bool ftFileSearch::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{ {
uint32_t hints, i; uint32_t hints, i;
#ifdef DEBUG_SEARCH #ifdef DEBUG_SEARCH
std::cerr << "ftFileSearch::search(" << hash << ", " << size; std::cerr << "ftFileSearch::search(" << hash ;
std::cerr << ", " << hintflags << ");"; std::cerr << ", " << hintflags << ");";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
@ -99,7 +99,7 @@ bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, F
std::cerr << i; std::cerr << i;
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
if (search->search(hash, size, hintflags, info)) if (search->search(hash, hintflags, info))
{ {
#ifdef DEBUG_SEARCH #ifdef DEBUG_SEARCH
std::cerr << "ftFileSearch::search() SLOT: "; std::cerr << "ftFileSearch::search() SLOT: ";
@ -155,7 +155,7 @@ bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, F
std::cerr << "ftFileSearch::search() SLOT: " << i; std::cerr << "ftFileSearch::search() SLOT: " << i;
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
if (search->search(hash, size, hintflags, info)) if (search->search(hash, hintflags, info))
{ {
#ifdef DEBUG_SEARCH #ifdef DEBUG_SEARCH
std::cerr << "ftFileSearch::search() SLOT: "; std::cerr << "ftFileSearch::search() SLOT: ";
@ -180,10 +180,10 @@ bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, F
} }
bool ftSearchDummy::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const bool ftSearchDummy::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{ {
#ifdef DEBUG_SEARCH #ifdef DEBUG_SEARCH
std::cerr << "ftSearchDummy::search(" << hash << ", " << size; std::cerr << "ftSearchDummy::search(" << hash ;
std::cerr << ", " << hintflags << ");"; std::cerr << ", " << hintflags << ");";
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif

2
libretroshare/src/ft/ftfilesearch.h
View File

@ -45,7 +45,7 @@ class ftFileSearch: public ftSearch
ftFileSearch(); ftFileSearch();
bool addSearchMode(ftSearch *search, uint32_t hintflags); bool addSearchMode(ftSearch *search, uint32_t hintflags);
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
private: private:

4
libretroshare/src/ft/ftsearch.h
View File

@ -43,7 +43,7 @@ class ftSearch
ftSearch() { return; } ftSearch() { return; }
virtual ~ftSearch() { return; } virtual ~ftSearch() { return; }
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const = 0; virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const = 0;
}; };
@ -54,7 +54,7 @@ class ftSearchDummy: public ftSearch
ftSearchDummy() { return; } ftSearchDummy() { return; }
virtual ~ftSearchDummy() { return; } virtual ~ftSearchDummy() { return; }
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
}; };
#endif #endif

49
libretroshare/src/ft/ftserver.cc
View File

@ -310,21 +310,9 @@ void ftServer::getDwlDetails(std::list<DwlDetails> & details)
mFtDwlQueue->getDwlDetails(details); mFtDwlQueue->getDwlDetails(details);
} }
bool ftServer::FileChunksDetails(const std::string& hash,FileChunksInfo& info) bool ftServer::FileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info)
{ {
return mFtController->getFileChunksDetails(hash,info); return mFtController->getFileDownloadChunksDetails(hash,info);
//
// // for know put some dummy info. It's for display sake only.
// info.chunk_size = 1024*1024 ;
// info.file_size = 250*info.chunk_size - 123 ; // last chunk is not complete.
// info.chunks.resize(250,FileChunksInfo::CHUNK_DONE) ;
// int n = rand()%150 + 50 ;
// for(int i=0;i<10;++i)
// info.chunks[n+i] = FileChunksInfo::CHUNK_ACTIVE ;
// for(int i=n+10;i<250;++i)
// info.chunks[i] = FileChunksInfo::CHUNK_OUTSTANDING ;
//
// return true ;
} }
/* Directory Handling */ /* Directory Handling */
@ -360,6 +348,11 @@ bool ftServer::FileDownloads(std::list<std::string> &hashs)
//return mFtDataplex->FileDownloads(hashs); //return mFtDataplex->FileDownloads(hashs);
} }
bool ftServer::FileUploadChunksDetails(const std::string& hash,const std::string& peer_id,CompressedChunkMap& cmap)
{
return mFtDataplex->getClientChunkMap(hash,peer_id,cmap);
}
bool ftServer::FileUploads(std::list<std::string> &hashs) bool ftServer::FileUploads(std::list<std::string> &hashs)
{ {
return mFtDataplex->FileUploads(hashs); return mFtDataplex->FileUploads(hashs);
@ -376,7 +369,7 @@ bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info)
return true ; return true ;
if(hintflags & ~(RS_FILE_HINTS_UPLOAD | RS_FILE_HINTS_DOWNLOAD)) if(hintflags & ~(RS_FILE_HINTS_UPLOAD | RS_FILE_HINTS_DOWNLOAD))
if(mFtSearch->search(hash, 0, hintflags, info)) if(mFtSearch->search(hash, hintflags, info))
return true ; return true ;
return false; return false;
@ -652,8 +645,7 @@ bool ftServer::loadConfigMap(std::map<std::string, std::string> &configMap)
/***************************************************************/ /***************************************************************/
/* Client Send */ /* Client Send */
bool ftServer::sendDataRequest(std::string peerId, std::string hash, bool ftServer::sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
uint64_t size, uint64_t offset, uint32_t chunksize)
{ {
if(mTurtleRouter->isTurtlePeer(peerId)) if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendDataRequest(peerId,hash,size,offset,chunksize) ; mTurtleRouter->sendDataRequest(peerId,hash,size,offset,chunksize) ;
@ -680,12 +672,33 @@ bool ftServer::sendDataRequest(std::string peerId, std::string hash,
return true; return true;
} }
bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string& hash)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendChunkMapRequest(peerId,hash) ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendChunkMap(const std::string& peerId,const std::string& hash,const CompressedChunkMap& map)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendChunkMap(peerId,hash,map) ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
//const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */ //const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */
//const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */ //const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */
const uint32_t MAX_FT_CHUNK = 8 * 1024; /* 16K */ const uint32_t MAX_FT_CHUNK = 8 * 1024; /* 16K */
/* Server Send */ /* Server Send */
bool ftServer::sendData(std::string peerId, std::string hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data) bool ftServer::sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data)
{ {
/* create a packet */ /* create a packet */
/* push to networking part */ /* push to networking part */

14
libretroshare/src/ft/ftserver.h
View File

@ -138,7 +138,9 @@ virtual void getDwlDetails(std::list<DwlDetails> & details);
virtual bool FileDownloads(std::list<std::string> &hashs); virtual bool FileDownloads(std::list<std::string> &hashs);
virtual bool FileUploads(std::list<std::string> &hashs); virtual bool FileUploads(std::list<std::string> &hashs);
virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info); virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info);
virtual bool FileChunksDetails(const std::string& hash,FileChunksInfo& info) ; virtual bool FileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info) ;
virtual bool FileUploadChunksDetails(const std::string& hash,const std::string& peer_id,CompressedChunkMap& map) ;
/*** /***
* Extra List Access * Extra List Access
@ -196,11 +198,11 @@ virtual bool unshareDownloadDirectory();
/*************** Data Transfer Interface ***********************/ /*************** Data Transfer Interface ***********************/
/***************************************************************/ /***************************************************************/
public: public:
virtual bool sendData(std::string peerId, std::string hash, uint64_t size, virtual bool sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
uint64_t offset, uint32_t chunksize, void *data); virtual bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
virtual bool sendDataRequest(std::string peerId, virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) ;
std::string hash, uint64_t size, virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) ;
uint64_t offset, uint32_t chunksize);
/*************** Internal Transfer Fns *************************/ /*************** Internal Transfer Fns *************************/
virtual int tick(); virtual int tick();

8
libretroshare/src/ft/fttransfermodule.cc
View File

@ -327,7 +327,11 @@ bool ftTransferModule::getChunk(const std::string& peer_id,uint32_t size_hint,ui
std::cerr << std::endl; std::cerr << std::endl;
#endif #endif
bool val = mFileCreator->getMissingChunk(peer_id,size_hint,offset, chunk_size); bool source_peer_map_needed ;
bool val = mFileCreator->getMissingChunk(peer_id,size_hint,offset, chunk_size,source_peer_map_needed);
if(source_peer_map_needed)
mMultiplexor->sendChunkMapRequest(peer_id, mHash) ;
#ifdef FT_DEBUG #ifdef FT_DEBUG
if (val) if (val)
@ -338,12 +342,14 @@ bool ftTransferModule::getChunk(const std::string& peer_id,uint32_t size_hint,ui
std::cerr << " size: " << mSize; std::cerr << " size: " << mSize;
std::cerr << " offset: " << offset; std::cerr << " offset: " << offset;
std::cerr << " chunk_size: " << chunk_size; std::cerr << " chunk_size: " << chunk_size;
std::cerr << " peer map needed = " << source_peer_map_needed << std::endl ;
std::cerr << std::endl; std::cerr << std::endl;
} }
else else
{ {
std::cerr << "ftTransferModule::getChunk()"; std::cerr << "ftTransferModule::getChunk()";
std::cerr << " Answer: No Chunk Available"; std::cerr << " Answer: No Chunk Available";
std::cerr << " peer map needed = " << source_peer_map_needed << std::endl ;
std::cerr << std::endl; std::cerr << std::endl;
} }
#endif #endif

7
libretroshare/src/rsiface/rsfiles.h
View File

@ -129,7 +129,12 @@ virtual void getDwlDetails(std::list<DwlDetails> & details) = 0;
virtual bool FileDownloads(std::list<std::string> &hashs) = 0; virtual bool FileDownloads(std::list<std::string> &hashs) = 0;
virtual bool FileUploads(std::list<std::string> &hashs) = 0; virtual bool FileUploads(std::list<std::string> &hashs) = 0;
virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0; virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0;
virtual bool FileChunksDetails(const std::string& hash,FileChunksInfo& info) = 0 ;
/// Gives chunk details about the downloaded file with given hash.
virtual bool FileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info) = 0 ;
/// details about the upload with given hash
virtual bool FileUploadChunksDetails(const std::string& hash,const std::string& peer_id,CompressedChunkMap& map) = 0 ;
/*** /***
* Extra List Access * Extra List Access

44
libretroshare/src/rsiface/rstypes.h
View File

@ -28,6 +28,7 @@
#include <list> #include <list>
#include <map>
#include <vector> #include <vector>
#include <iostream> #include <iostream>
#include <string> #include <string>
@ -250,7 +251,10 @@ enum DwlPriority { Low = 0, Normal, High, Auto };
// Macro to read a bits array for compressed chunk maps // Macro to read a bits array for compressed chunk maps
// //
#define COMPRESSED_MAP_READ(A,j) (A[j >> 5] & (1 << (j & 0x11111))) //#define COMPRESSED_MAP_READ(A,j) (A[j >> 5] & (1 << (j & 0x11111)))
//#define COMPRESSED_MAP_WRITE(A,j,x) (A[j >> 5] |= (1 << (j & 0x11111)))
class CompressedChunkMap ;
class FileChunksInfo class FileChunksInfo
{ {
@ -265,12 +269,48 @@ class FileChunksInfo
std::vector<ChunkState> chunks ; std::vector<ChunkState> chunks ;
// For each source peer, gives the compressed bit map of have/don't have sate // For each source peer, gives the compressed bit map of have/don't have sate
std::vector<std::pair<std::string, std::vector<uint32_t> > > compressed_peer_availability_maps ; std::map<std::string, CompressedChunkMap> compressed_peer_availability_maps ;
// For each chunk (by chunk number), gives the completion of the chunk. // For each chunk (by chunk number), gives the completion of the chunk.
// //
std::vector<std::pair<uint32_t,uint32_t> > active_chunks ; std::vector<std::pair<uint32_t,uint32_t> > active_chunks ;
};
class CompressedChunkMap
{
public:
CompressedChunkMap() {}
CompressedChunkMap(const std::vector<FileChunksInfo::ChunkState>& uncompressed_data)
{
_map.resize( getCompressedSize(uncompressed_data.size()),0 ) ;
for(uint32_t i=0;i<uncompressed_data.size();++i)
if(uncompressed_data[i]==FileChunksInfo::CHUNK_DONE)
set(i) ;
}
CompressedChunkMap(uint32_t nb_chunks,uint32_t value)
{
_map.resize(getCompressedSize(nb_chunks),value) ;
}
static uint32_t getCompressedSize(uint32_t size) { return (size>>5) + !!(size&31) ; }
uint32_t filledChunks(uint32_t nbchks)
{
uint32_t res = 0 ;
for(uint32_t i=0;i<std::min(nbchks,_map.size()*32);++i)
res += operator[](i) ;
return res ;
}
inline bool operator[](uint32_t i) const { return (_map[i >> 5] & (1 << (i & 31))) > 0 ; }
inline void set(uint32_t j) { _map[j >> 5] |= (1 << (j & 31)) ; }
inline void reset(uint32_t j) { _map[j >> 5] &= ~(1 << (j & 31)) ; }
/// compressed map, one bit per chunk
std::vector<uint32_t> _map ;
}; };
/* class which encapsulates download details */ /* class which encapsulates download details */

18
libretroshare/src/serialiser/rsconfigitems.cc
View File

@ -196,11 +196,9 @@ uint32_t RsFileConfigSerialiser::sizeTransfer(RsFileTransfer *item)
s += 4; /* trate */ s += 4; /* trate */
s += 4; /* lrate */ s += 4; /* lrate */
s += 4; /* ltransfer */ s += 4; /* ltransfer */
s += 4; // chunk_size
s += 4; // chunk_number
s += 4; // chunk_strategy s += 4; // chunk_strategy
s += 4; // chunk map size s += 4; // chunk map size
s += 4*item->chunk_map.size(); // chunk_map s += 4*item->compressed_chunk_map._map.size(); // compressed_chunk_map
return s; return s;
} }
@ -243,13 +241,11 @@ bool RsFileConfigSerialiser::serialiseTransfer(RsFileTransfer *item, void *d
ok &= setRawUInt32(data, tlvsize, &offset, item->lrate); ok &= setRawUInt32(data, tlvsize, &offset, item->lrate);
ok &= setRawUInt32(data, tlvsize, &offset, item->ltransfer); ok &= setRawUInt32(data, tlvsize, &offset, item->ltransfer);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_size);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_number);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_strategy); ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_strategy);
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map.size()); ok &= setRawUInt32(data, tlvsize, &offset, item->compressed_chunk_map._map.size());
for(uint32_t i=0;i<item->chunk_map.size();++i) for(uint32_t i=0;i<item->compressed_chunk_map._map.size();++i)
ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map[i]); ok &= setRawUInt32(data, tlvsize, &offset, item->compressed_chunk_map._map[i]);
if (offset != tlvsize) if (offset != tlvsize)
{ {
@ -310,15 +306,13 @@ RsFileTransfer *RsFileConfigSerialiser::deserialiseTransfer(void *data, uint32_t
ok &= getRawUInt32(data, rssize, &offset, &(item->lrate)); ok &= getRawUInt32(data, rssize, &offset, &(item->lrate));
ok &= getRawUInt32(data, rssize, &offset, &(item->ltransfer)); ok &= getRawUInt32(data, rssize, &offset, &(item->ltransfer));
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_size));
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_number));
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_strategy)); ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_strategy));
uint32_t map_size = 0 ; uint32_t map_size = 0 ;
ok &= getRawUInt32(data, rssize, &offset, &map_size); ok &= getRawUInt32(data, rssize, &offset, &map_size);
item->chunk_map.resize(map_size) ; item->compressed_chunk_map._map.resize(map_size) ;
for(uint32_t i=0;i<map_size;++i) for(uint32_t i=0;i<map_size;++i)
ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_map[i])); ok &= getRawUInt32(data, rssize, &offset, &(item->compressed_chunk_map._map[i]));
if (offset != rssize) if (offset != rssize)
{ {

5
libretroshare/src/serialiser/rsconfigitems.h
View File

@ -29,6 +29,7 @@
#include <map> #include <map>
#include <vector> #include <vector>
#include <rsiface/rstypes.h>
#include "serialiser/rsserial.h" #include "serialiser/rsserial.h"
#include "serialiser/rstlvbase.h" #include "serialiser/rstlvbase.h"
#include "serialiser/rstlvtypes.h" #include "serialiser/rstlvtypes.h"
@ -204,10 +205,8 @@ class RsFileTransfer: public RsItem
uint32_t ltransfer; uint32_t ltransfer;
// chunk information // chunk information
uint32_t chunk_size ; // common size of chunks
uint32_t chunk_number ; // total number of chunks (this is not redondant, cause chunks are compressed)
uint32_t chunk_strategy ; // strategy flags for chunks uint32_t chunk_strategy ; // strategy flags for chunks
std::vector<uint32_t> chunk_map ; // chunk availability (bitwise) CompressedChunkMap compressed_chunk_map ; // chunk availability (bitwise)
}; };
/**************************************************************************/ /**************************************************************************/

183
libretroshare/src/turtle/p3turtle.cc
View File

@ -782,25 +782,24 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
// The packet was not forwarded, so it is for us. Let's treat it. // The packet was not forwarded, so it is for us. Let's treat it.
// This is done off-mutex, to avoid various deadlocks // This is done off-mutex, to avoid various deadlocks
// //
if(direction == RsTurtleGenericTunnelItem::DIRECTION_SERVER)
switch(item->PacketSubType()) switch(item->PacketSubType())
{ {
case RS_TURTLE_SUBTYPE_FILE_REQUEST: handleRecvFileRequest(dynamic_cast<RsTurtleFileRequestItem *>(item)) ; case RS_TURTLE_SUBTYPE_FILE_REQUEST: handleRecvFileRequest(dynamic_cast<RsTurtleFileRequestItem *>(item)) ;
break ; break ;
default:
std::cerr << "Unknown server packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ;
exit(-1) ;
}
if(direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) case RS_TURTLE_SUBTYPE_FILE_DATA : handleRecvFileData(dynamic_cast<RsTurtleFileDataItem *>(item)) ;
switch(item->PacketSubType()) break ;
{
case RS_TURTLE_SUBTYPE_FILE_DATA : handleRecvFileData(dynamic_cast<RsTurtleFileDataItem *>(item)) ; case RS_TURTLE_SUBTYPE_FILE_MAP : handleRecvFileMap(dynamic_cast<RsTurtleFileMapItem *>(item)) ;
break ; break ;
default:
std::cerr << "Unknown client packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ; case RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST: handleRecvFileMapRequest(dynamic_cast<RsTurtleFileMapRequestItem *>(item)) ;
exit(-1) ; break ;
} default:
std::cerr << "Unknown packet type received: id=" << (void*)(item->PacketSubType()) << std::endl ;
exit(-1) ;
}
delete item ; delete item ;
} }
@ -908,6 +907,42 @@ void p3turtle::handleRecvFileData(RsTurtleFileDataItem *item)
// down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData // down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData
} }
void p3turtle::handleRecvFileMapRequest(RsTurtleFileMapRequestItem *item)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3Turtle: received file Map item:" << std::endl ;
item->print(std::cerr,1) ;
#endif
std::string hash,vpid ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2(_local_tunnels.find(item->tunnel_id)) ;
if(it2 == _local_tunnels.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: got file data with unknown tunnel id " << (void*)item->tunnel_id << std::endl ;
#endif
return ;
}
TurtleTunnel& tunnel(it2->second) ;
#ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ;
std::cerr << " This is an endpoint for this file map request." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif
// we should check that there is no backward call to the turtle router!
//
hash = tunnel.hash ;
vpid = tunnel.vpid ;
}
_ft_server->getMultiplexer()->recvChunkMapRequest(vpid,hash,item->direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ;
}
void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item) void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
{ {
@ -915,6 +950,7 @@ void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
std::cerr << "p3Turtle: received file Map item:" << std::endl ; std::cerr << "p3Turtle: received file Map item:" << std::endl ;
item->print(std::cerr,1) ; item->print(std::cerr,1) ;
#endif #endif
std::string hash,vpid ;
{ {
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
@ -930,31 +966,19 @@ void p3turtle::handleRecvFileMap(RsTurtleFileMapItem *item)
TurtleTunnel& tunnel(it2->second) ; TurtleTunnel& tunnel(it2->second) ;
std::map<TurtleFileHash,TurtleFileHashInfo>::iterator it( _incoming_file_hashes.find(tunnel.hash) ) ;
#ifdef P3TURTLE_DEBUG #ifdef P3TURTLE_DEBUG
assert(!tunnel.hash.empty()) ; assert(!tunnel.hash.empty()) ;
#endif
if(it==_incoming_file_hashes.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "No tunnel for incoming data. Maybe the tunnel is being closed." << std::endl ;
#endif
return ;
}
const TurtleFileHashInfo& hash_info(it->second) ;
#ifdef P3TURTLE_DEBUG
std::cerr << " This is an endpoint for this file map." << std::endl ; std::cerr << " This is an endpoint for this file map." << std::endl ;
std::cerr << " Forwarding data to the multiplexer." << std::endl ; std::cerr << " Forwarding data to the multiplexer." << std::endl ;
std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ; std::cerr << " using peer_id=" << tunnel.vpid << ", hash=" << tunnel.hash << std::endl ;
#endif #endif
// also update the hash time stamp to show that it's actually being downloaded. // We should check that there is no backward call to the turtle router!
it->second.time_stamp = time(NULL) ;
// we should check that there is no backward call to the turtle router!
// //
_ft_server->getMultiplexer()->recvFileMap(tunnel.vpid,tunnel.hash,item->chunk_size,item->nb_chunks,item->compressed_map) ; vpid = tunnel.vpid ;
hash = tunnel.hash ;
} }
_ft_server->getMultiplexer()->recvChunkMap(vpid,hash,item->compressed_map,item->direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ;
} }
// Send a data request into the correct tunnel for the given file hash // Send a data request into the correct tunnel for the given file hash
@ -1032,6 +1056,95 @@ void p3turtle::sendFileData(const std::string& peerId, const std::string& hash,
sendItem(item) ; sendItem(item) ;
} }
void p3turtle::sendChunkMapRequest(const std::string& peerId,const std::string& hash)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::senddataRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleFileMapRequestItem *item = new RsTurtleFileMapRequestItem ;
item->tunnel_id = tunnel_id ;
std::string ownid = mConnMgr->getOwnId() ;
if(tunnel.local_src == ownid)
{
item->direction = RsTurtleGenericTunnelItem::DIRECTION_SERVER ;
item->PeerId(tunnel.local_dst) ;
}
else if(tunnel.local_dst == ownid)
{
item->direction = RsTurtleGenericTunnelItem::DIRECTION_CLIENT ;
item->PeerId(tunnel.local_src) ;
}
else
std::cerr << "p3turtle::sendChunkMap: consistency error!" << std::endl ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending chunk map req to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
void p3turtle::sendChunkMap(const std::string& peerId,const std::string& hash,const CompressedChunkMap& cmap)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// get the proper tunnel for this file hash and peer id.
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(peerId)) ;
if(it == _virtual_peers.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle::senddataRequest: cannot find virtual peer " << peerId << " in VP list." << std::endl ;
#endif
return ;
}
TurtleTunnelId tunnel_id = it->second ;
TurtleTunnel& tunnel(_local_tunnels[tunnel_id]) ;
#ifdef P3TURTLE_DEBUG
assert(hash == tunnel.hash) ;
#endif
RsTurtleFileMapItem *item = new RsTurtleFileMapItem ;
item->tunnel_id = tunnel_id ;
item->compressed_map = cmap ;
std::string ownid = mConnMgr->getOwnId() ;
if(tunnel.local_src == ownid)
{
item->direction = RsTurtleGenericTunnelItem::DIRECTION_SERVER ;
item->PeerId(tunnel.local_dst) ;
}
else if(tunnel.local_dst == ownid)
{
item->direction = RsTurtleGenericTunnelItem::DIRECTION_CLIENT ;
item->PeerId(tunnel.local_src) ;
}
else
std::cerr << "p3turtle::sendChunkMap: consistency error!" << std::endl ;
#ifdef P3TURTLE_DEBUG
std::cerr << "p3turtle: sending chunk map to peer " << peerId << ", hash=0x" << hash << ") through tunnel " << (void*)item->tunnel_id << ", next peer=" << item->PeerId() << std::endl ;
#endif
sendItem(item) ;
}
// This function is actually not needed: Search request to the turtle router are: // This function is actually not needed: Search request to the turtle router are:
// - distant search requests, handled by the router // - distant search requests, handled by the router
// - search requests over files being downloaded, handled by rsFiles !! // - search requests over files being downloaded, handled by rsFiles !!
@ -1573,7 +1686,7 @@ static std::string printNumber(uint64_t num,bool hex=false)
if(hex) if(hex)
{ {
char tmp[100] ; char tmp[100] ;
sprintf(tmp,"0x%08x%08x", uint32_t(num >> 32),uint32_t(num & ( (1<<32)-1 ))) ; sprintf(tmp,"%08x%08x", uint32_t(num >> 32),uint32_t(num & ( (1<<32)-1 ))) ;
return std::string(tmp) ; return std::string(tmp) ;
} }
else else

8
libretroshare/src/turtle/p3turtle.h
View File

@ -280,6 +280,13 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f
/// Send file data into the correct tunnel for the given file hash /// Send file data into the correct tunnel for the given file hash
void sendFileData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data) ; void sendFileData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data) ;
/// Send a request for the chunk map of this file to the given peer
void sendChunkMapRequest(const std::string& peerId, const std::string& hash) ;
/// Send a chunk map of this file to the given peer
void sendChunkMap(const std::string& peerId, const std::string& hash,const CompressedChunkMap& cmap) ;
private: private:
//--------------------------- Admin/Helper functions -------------------------// //--------------------------- Admin/Helper functions -------------------------//
@ -321,6 +328,7 @@ class p3turtle: public p3Service, public pqiMonitor, public RsTurtle,/* public f
void handleTunnelResult(RsTurtleTunnelOkItem *item); void handleTunnelResult(RsTurtleTunnelOkItem *item);
void handleRecvFileRequest(RsTurtleFileRequestItem *item); void handleRecvFileRequest(RsTurtleFileRequestItem *item);
void handleRecvFileData(RsTurtleFileDataItem *item); void handleRecvFileData(RsTurtleFileDataItem *item);
void handleRecvFileMapRequest(RsTurtleFileMapRequestItem*);
void handleRecvFileMap(RsTurtleFileMapItem*); void handleRecvFileMap(RsTurtleFileMapItem*);
//------ Functions connecting the turtle router to other components.----------// //------ Functions connecting the turtle router to other components.----------//

120
libretroshare/src/turtle/rsturtleitem.cc
View File

@ -114,17 +114,27 @@ uint32_t RsTurtleFileDataItem::serial_size()
return s ; return s ;
} }
uint32_t RsTurtleFileMapRequestItem::serial_size()
{
uint32_t s = 0 ;
s += 8 ; // header
s += 4 ; // tunnel id
s += 4 ; // direction
return s ;
}
uint32_t RsTurtleFileMapItem::serial_size() uint32_t RsTurtleFileMapItem::serial_size()
{ {
uint32_t s = 0 ; uint32_t s = 0 ;
s += 8 ; // header s += 8 ; // header
s += 4 ; // tunnel id s += 4 ; // tunnel id
s += 4 ; // chunk_size s += 4 ; // direction
s += 4 ; // nb_chunks
s += 4 ; // compressed_map.size() s += 4 ; // compressed_map.size()
s += 4 * compressed_map.size() ; s += 4 * compressed_map._map.size() ;
return s ; return s ;
} }
@ -162,6 +172,7 @@ RsItem *RsTurtleSerialiser::deserialise(void *data, uint32_t *size)
case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem(data,*size) ; case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_REQUEST : return new RsTurtleFileRequestItem(data,*size) ; case RS_TURTLE_SUBTYPE_FILE_REQUEST : return new RsTurtleFileRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_DATA : return new RsTurtleFileDataItem(data,*size) ; case RS_TURTLE_SUBTYPE_FILE_DATA : return new RsTurtleFileDataItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST : return new RsTurtleFileMapRequestItem(data,*size) ;
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem(data,*size) ; case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem(data,*size) ;
default: default:
@ -179,6 +190,39 @@ RsItem *RsTurtleSerialiser::deserialise(void *data, uint32_t *size)
} }
bool RsTurtleFileMapRequestItem::serialize(void *data,uint32_t& pktsize)
{
uint32_t tlvsize = serial_size();
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data,tlvsize,PacketId(), tlvsize);
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, direction);
if (offset != tlvsize)
{
ok = false;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileConfigSerialiser::serialiseTransfer() Size Error! " << std::endl;
#endif
}
return ok;
}
bool RsTurtleFileMapItem::serialize(void *data,uint32_t& pktsize) bool RsTurtleFileMapItem::serialize(void *data,uint32_t& pktsize)
{ {
uint32_t tlvsize = serial_size(); uint32_t tlvsize = serial_size();
@ -199,12 +243,11 @@ bool RsTurtleFileMapItem::serialize(void *data,uint32_t& pktsize)
/* add mandatory parts first */ /* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id); ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id);
ok &= setRawUInt32(data, tlvsize, &offset, chunk_size); ok &= setRawUInt32(data, tlvsize, &offset, direction);
ok &= setRawUInt32(data, tlvsize, &offset, nb_chunks); ok &= setRawUInt32(data, tlvsize, &offset, compressed_map._map.size());
ok &= setRawUInt32(data, tlvsize, &offset, compressed_map.size());
for(uint32_t i=0;i<compressed_map.size() && ok;++i) for(uint32_t i=0;i<compressed_map._map.size() && ok;++i)
ok &= setRawUInt32(data, tlvsize, &offset, compressed_map[i]); ok &= setRawUInt32(data, tlvsize, &offset, compressed_map._map[i]);
if (offset != tlvsize) if (offset != tlvsize)
{ {
@ -409,24 +452,54 @@ RsTurtleFileMapItem::RsTurtleFileMapItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP) : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP)
{ {
#ifdef P3TURTLE_DEBUG #ifdef P3TURTLE_DEBUG
std::cerr << " type = search result" << std::endl ; std::cerr << " type = file map item" << std::endl ;
#endif #endif
uint32_t offset = 8; // skip the header uint32_t offset = 8; // skip the header
uint32_t rssize = getRsItemSize(data);
/* add mandatory parts first */ /* add mandatory parts first */
bool ok = true ; bool ok = true ;
uint32_t s ; uint32_t s,d ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id); ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id);
ok &= getRawUInt32(data, pktsize, &offset, &chunk_size); ok &= getRawUInt32(data, pktsize, &offset, &d);
ok &= getRawUInt32(data, pktsize, &offset, &nb_chunks) ; direction = d ;
ok &= getRawUInt32(data, pktsize, &offset, &s) ; ok &= getRawUInt32(data, pktsize, &offset, &s) ;
compressed_map.resize(s) ; compressed_map._map.resize(s) ;
for(uint32_t i=0;i<s && ok;++i) for(uint32_t i=0;i<s && ok;++i)
ok &= getRawUInt32(data, pktsize, &offset, &(compressed_map[i])) ; ok &= getRawUInt32(data, pktsize, &offset, &(compressed_map._map[i])) ;
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
}
RsTurtleFileMapRequestItem::RsTurtleFileMapRequestItem(void *data,uint32_t pktsize)
: RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " type = file map request item" << std::endl ;
#endif
uint32_t offset = 8; // skip the header
/* add mandatory parts first */
bool ok = true ;
ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id);
ok &= getRawUInt32(data, pktsize, &offset, &direction);
#ifdef WINDOWS_SYS // No Exceptions in Windows compile. (drbobs).
#else
if (offset != pktsize)
throw std::runtime_error("Size error while deserializing.") ;
if (!ok)
throw std::runtime_error("Unknown error while deserializing.") ;
#endif
} }
RsTurtleSearchResultItem::RsTurtleSearchResultItem(void *data,uint32_t pktsize) RsTurtleSearchResultItem::RsTurtleSearchResultItem(void *data,uint32_t pktsize)
@ -826,12 +899,21 @@ std::ostream& RsTurtleFileMapItem::print(std::ostream& o, uint16_t)
o << "File map item:" << std::endl ; o << "File map item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ; o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " chunk size: " << chunk_size << std::endl ; o << " direction : " << direction << std::endl ;
o << " nb chunks : " << nb_chunks << std::endl ;
o << " map : " ; o << " map : " ;
for(uint32_t i=0;i<compressed_map.size();++i) for(uint32_t i=0;i<compressed_map._map.size();++i)
o << (void*)compressed_map[i] << std::endl ; o << (void*)compressed_map._map[i] << std::endl ;
return o ;
}
std::ostream& RsTurtleFileMapRequestItem::print(std::ostream& o, uint16_t)
{
o << "File map request item:" << std::endl ;
o << " tunnel id : " << (void*)tunnel_id << std::endl ;
o << " direction : " << direction << std::endl ;
return o ; return o ;
} }

30
libretroshare/src/turtle/rsturtleitem.h
View File

@ -5,6 +5,7 @@
#include "serialiser/rsbaseserial.h" #include "serialiser/rsbaseserial.h"
#include "rsiface/rsturtle.h" #include "rsiface/rsturtle.h"
#include "rsiface/rsexpr.h" #include "rsiface/rsexpr.h"
#include "rsiface/rstypes.h"
#include "serialiser/rsserviceids.h" #include "serialiser/rsserviceids.h"
#include "turtle/turtletypes.h" #include "turtle/turtletypes.h"
@ -18,6 +19,7 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST = 0x07 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ;
const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ; const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ;
/***********************************************************************************/ /***********************************************************************************/
/* Basic Turtle Item Class */ /* Basic Turtle Item Class */
@ -153,7 +155,9 @@ class RsTurtleGenericTunnelItem: public RsTurtleItem
public: public:
RsTurtleGenericTunnelItem(uint8_t sub_packet_id) : RsTurtleItem(sub_packet_id) {} RsTurtleGenericTunnelItem(uint8_t sub_packet_id) : RsTurtleItem(sub_packet_id) {}
typedef enum { DIRECTION_CLIENT, DIRECTION_SERVER } Direction ; typedef uint32_t Direction ;
static const Direction DIRECTION_CLIENT = 0x001 ;
static const Direction DIRECTION_SERVER = 0x002 ;
/// Does this packet stamps tunnels when it passes through ? /// Does this packet stamps tunnels when it passes through ?
/// This is used for keeping trace weither tunnels are active or not. /// This is used for keeping trace weither tunnels are active or not.
@ -218,6 +222,26 @@ class RsTurtleFileDataItem: public RsTurtleGenericTunnelItem
virtual uint32_t serial_size() ; virtual uint32_t serial_size() ;
}; };
class RsTurtleFileMapRequestItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleFileMapRequestItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST) {}
RsTurtleFileMapRequestItem(void *data,uint32_t size) ; // deserialization
virtual bool shouldStampTunnel() const { return false ; }
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
virtual Direction travelingDirection() const { return direction ; }
Direction direction ; // travel direction for this packet (server/client)
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
// this info from the file size, but this allows a security check.
virtual std::ostream& print(std::ostream& o, uint16_t) ;
virtual bool serialize(void *data,uint32_t& size) ;
virtual uint32_t serial_size() ;
};
class RsTurtleFileMapItem: public RsTurtleGenericTunnelItem class RsTurtleFileMapItem: public RsTurtleGenericTunnelItem
{ {
public: public:
@ -230,11 +254,9 @@ class RsTurtleFileMapItem: public RsTurtleGenericTunnelItem
Direction direction ; // travel direction for this packet (server/client) Direction direction ; // travel direction for this packet (server/client)
uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source uint32_t tunnel_id ; // id of the tunnel to travel through. Also used for identifying the file source
uint32_t chunk_size ; // fixed size of chunks, as seen from the source, for the given map.
uint32_t nb_chunks ; // number of chunks in the file. The last two infos are redundant, as we can recompute
// this info from the file size, but this allows a security check. // this info from the file size, but this allows a security check.
std::vector<uint32_t> compressed_map ; // Map info for the file in compressed format. Each *bit* in the array uint's says "I have" or "I don't have" CompressedChunkMap compressed_map ; // Map info for the file in compressed format. Each *bit* in the array uint's says "I have" or "I don't have"
// by default, we suppose the peer has all the chunks. This info will thus be and-ed // by default, we suppose the peer has all the chunks. This info will thus be and-ed
// with the default file map for this source. // with the default file map for this source.

19
retroshare-gui/src/gui/DLListDelegate.cpp
View File

@ -19,6 +19,7 @@
* Boston, MA 02110-1301, USA. * Boston, MA 02110-1301, USA.
****************************************************************/ ****************************************************************/
#include <rsiface/rstypes.h>
#include <QModelIndex> #include <QModelIndex>
#include <QPainter> #include <QPainter>
#include <QStyleOptionProgressBarV2> #include <QStyleOptionProgressBarV2>
@ -27,6 +28,8 @@
#include "DLListDelegate.h" #include "DLListDelegate.h"
Q_DECLARE_METATYPE(FileProgressInfo)
DLListDelegate::DLListDelegate(QObject *parent) : QAbstractItemDelegate(parent) DLListDelegate::DLListDelegate(QObject *parent) : QAbstractItemDelegate(parent)
{ {
; ;
@ -144,14 +147,14 @@ void DLListDelegate::paint(QPainter * painter, const QStyleOptionViewItem & opti
painter->drawText(option.rect, Qt::AlignRight, temp); painter->drawText(option.rect, Qt::AlignRight, temp);
break; break;
case PROGRESS: case PROGRESS:
{ {
progress = index.data().toDouble(); // create a xProgressBar
// create a xProgressBar FileProgressInfo pinfo = index.data().value<FileProgressInfo>() ;
xProgressBar progressBar(option.rect, painter); // the 3rd param is the color schema (0 is the default value) xProgressBar progressBar(pinfo.cmap,option.rect, painter); // the 3rd param is the color schema (0 is the default value)
progressBar.setDisplayText(false); // should display % text? progressBar.setDisplayText(false); // should display % text?
progressBar.setValue(progress); // set the progress value progressBar.setValue(pinfo.progress); // set the progress value
progressBar.setVerticalSpan(1); progressBar.setVerticalSpan(1);
progressBar.paint(); // paint the progress bar progressBar.paint(); // paint the progress bar
} }
painter->drawText(option.rect, Qt::AlignCenter, newopt.text); painter->drawText(option.rect, Qt::AlignCenter, newopt.text);
break; break;

30
retroshare-gui/src/gui/FileTransferInfoWidget.cpp
View File

@ -62,23 +62,16 @@ void FileTransferInfoWidget::updateDisplay()
{ {
std::cout << "In TaskGraphPainterWidget::updateDisplay()" << std::endl ; std::cout << "In TaskGraphPainterWidget::updateDisplay()" << std::endl ;
bool ok=true ;
FileInfo nfo ; FileInfo nfo ;
if(!rsFiles->FileDetails(_file_hash, RS_FILE_HINTS_DOWNLOAD, nfo)) if(!rsFiles->FileDetails(_file_hash, RS_FILE_HINTS_DOWNLOAD, nfo))
return ; ok = false ;
FileChunksInfo info ; FileChunksInfo info ;
if(!rsFiles->FileChunksDetails(_file_hash, info)) if(!rsFiles->FileDownloadChunksDetails(_file_hash, info))
return ; ok = false ;
std::cout << "got details for file " << nfo.fname << std::endl ; std::cout << "got details for file " << nfo.fname << std::endl ;
uint64_t fileSize = info.file_size;
uint32_t blockSize = info.chunk_size ;
int blocks = info.chunks.size() ;
int columns = maxWidth/chunk_square_size;
y = blocks/columns*chunk_square_size;
x = blocks%columns*chunk_square_size;
maxHeight = y+150+info.active_chunks.size()*(block_sep+text_height); // warning: this should be computed from the different size parameter and the number of objects drawn, otherwise the last objects to be displayed will be truncated.
pixmap = QPixmap(size()); pixmap = QPixmap(size());
pixmap.fill(this, 0, 0); pixmap.fill(this, 0, 0);
pixmap = QPixmap(maxWidth, maxHeight); pixmap = QPixmap(maxWidth, maxHeight);
@ -88,7 +81,16 @@ void FileTransferInfoWidget::updateDisplay()
QPainter painter(&pixmap); QPainter painter(&pixmap);
painter.initFrom(this); painter.initFrom(this);
draw(info,&painter) ; if(ok)
{
int blocks = info.chunks.size() ;
int columns = maxWidth/chunk_square_size;
y = blocks/columns*chunk_square_size;
x = blocks%columns*chunk_square_size;
maxHeight = y+150+info.active_chunks.size()*(block_sep+text_height); // warning: this should be computed from the different size parameter and the number of objects drawn, otherwise the last objects to be displayed will be truncated.
draw(info,&painter) ;
}
pixmap2 = pixmap; pixmap2 = pixmap;
} }
@ -194,8 +196,8 @@ void FileTransferInfoWidget::draw(const FileChunksInfo& info,QPainter *painter)
int nb_src = 0 ; int nb_src = 0 ;
int chunk_num = (int)floor(i/float(availability_map_size_X)*(nb_chunks-1)) ; int chunk_num = (int)floor(i/float(availability_map_size_X)*(nb_chunks-1)) ;
for(uint j=0;j<info.compressed_peer_availability_maps.size();++j) for(std::map<std::string,CompressedChunkMap>::const_iterator it(info.compressed_peer_availability_maps.begin());it!=info.compressed_peer_availability_maps.end();++it)
nb_src += (bool)(COMPRESSED_MAP_READ(info.compressed_peer_availability_maps[j].second, chunk_num)) ; nb_src += it->second[chunk_num] ;
painter->setPen(QColor::fromHsv(200,50*nb_src,200)) ; // the more sources, the more saturated painter->setPen(QColor::fromHsv(200,50*nb_src,200)) ; // the more sources, the more saturated
painter->drawLine(i,y,i,y+availability_map_size_Y) ; painter->drawLine(i,y,i,y+availability_map_size_Y) ;

135
retroshare-gui/src/gui/TransfersDialog.cpp
View File

@ -31,6 +31,7 @@
#include "DLListDelegate.h" #include "DLListDelegate.h"
#include "ULListDelegate.h" #include "ULListDelegate.h"
#include "FileTransferInfoWidget.h" #include "FileTransferInfoWidget.h"
#include "xprogressbar.h"
#include <QContextMenuEvent> #include <QContextMenuEvent>
#include <QMenu> #include <QMenu>
@ -69,6 +70,34 @@
#define IMAGE_PRIORITYHIGH ":/images/priorityhigh.png" #define IMAGE_PRIORITYHIGH ":/images/priorityhigh.png"
#define IMAGE_PRIORITYAUTO ":/images/priorityauto.png" #define IMAGE_PRIORITYAUTO ":/images/priorityauto.png"
//static const CompressedChunkMap *getMap_tmp()
//{
// static CompressedChunkMap *cmap = NULL ;
//
// if(cmap == NULL)
// {
// cmap = new CompressedChunkMap ; // to be passed as a parameter
// // Initialize the chunkmap with a dummy value, for testing. To be removed...
// cmap->_nb_chunks = 700 ;
// cmap->_map.resize(cmap->_nb_chunks/32+1,0) ;
// cmap->_progress = 0.34 ;
// for(uint i=0;i<10;++i)
// {
// uint32_t start = rand()%cmap->_nb_chunks;
// uint32_t j = std::min((int)cmap->_nb_chunks,(int)start+10+(rand()%5)) ;
//
// for(uint32_t k=start;k<j;++k)
// COMPRESSED_MAP_WRITE(cmap->_map,k,1) ;
// }
//
// std::cerr << "Built cmap = " << (void*)cmap << std::endl ;
// }
// return cmap ;
//}
Q_DECLARE_METATYPE(FileProgressInfo)
/** Constructor */ /** Constructor */
TransfersDialog::TransfersDialog(QWidget *parent) TransfersDialog::TransfersDialog(QWidget *parent)
@ -378,11 +407,11 @@ void TransfersDialog::playSelectedTransfer()
void TransfersDialog::updateProgress(int value) void TransfersDialog::updateProgress(int value)
{ {
for(int i = 0; i <= DLListModel->rowCount(); i++) { // for(int i = 0; i <= DLListModel->rowCount(); i++) {
if(selection->isRowSelected(i, QModelIndex())) { // if(selection->isRowSelected(i, QModelIndex())) {
editItem(i, PROGRESS, QVariant((double)value)); // editItem(i, PROGRESS, QVariant((double)value));
} // }
} // }
} }
TransfersDialog::~TransfersDialog() TransfersDialog::~TransfersDialog()
@ -391,8 +420,7 @@ TransfersDialog::~TransfersDialog()
} }
int TransfersDialog::addItem(QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed,
int TransfersDialog::addItem(QString symbol, QString name, QString coreID, qlonglong fileSize, double progress, double dlspeed,
QString sources, QString status, QString priority, qlonglong completed, qlonglong remaining) QString sources, QString status, QString priority, qlonglong completed, qlonglong remaining)
{ {
int row; int row;
@ -408,7 +436,7 @@ int TransfersDialog::addItem(QString symbol, QString name, QString coreID, qlong
DLListModel->setData(DLListModel->index(row, SIZE), QVariant((qlonglong)fileSize)); DLListModel->setData(DLListModel->index(row, SIZE), QVariant((qlonglong)fileSize));
DLListModel->setData(DLListModel->index(row, COMPLETED), QVariant((qlonglong)completed)); DLListModel->setData(DLListModel->index(row, COMPLETED), QVariant((qlonglong)completed));
DLListModel->setData(DLListModel->index(row, DLSPEED), QVariant((double)dlspeed)); DLListModel->setData(DLListModel->index(row, DLSPEED), QVariant((double)dlspeed));
DLListModel->setData(DLListModel->index(row, PROGRESS), QVariant((double)progress)); DLListModel->setData(DLListModel->index(row, PROGRESS), QVariant::fromValue(pinfo));
DLListModel->setData(DLListModel->index(row, SOURCES), QVariant((QString)sources)); DLListModel->setData(DLListModel->index(row, SOURCES), QVariant((QString)sources));
DLListModel->setData(DLListModel->index(row, STATUS), QVariant((QString)status)); DLListModel->setData(DLListModel->index(row, STATUS), QVariant((QString)status));
DLListModel->setData(DLListModel->index(row, PRIORITY), QVariant((QString)priority)); DLListModel->setData(DLListModel->index(row, PRIORITY), QVariant((QString)priority));
@ -469,7 +497,7 @@ int TransfersDialog::addItem(QString symbol, QString name, QString coreID, qlong
return row; return row;
} }
bool TransfersDialog::addPeerToItem(int row, QString symbol, QString name, QString coreID, qlonglong fileSize, double progress, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining) bool TransfersDialog::addPeerToItem(int row, QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining)
{ {
QStandardItem *dlItem = DLListModel->item(row); QStandardItem *dlItem = DLListModel->item(row);
if (!dlItem) return false; if (!dlItem) return false;
@ -484,7 +512,7 @@ bool TransfersDialog::addPeerToItem(int row, QString symbol, QString name, QStri
QStandardItem *i2 = new QStandardItem(); i2->setData(QVariant((qlonglong)fileSize), Qt::DisplayRole); QStandardItem *i2 = new QStandardItem(); i2->setData(QVariant((qlonglong)fileSize), Qt::DisplayRole);
QStandardItem *i3 = new QStandardItem(); i3->setData(QVariant((qlonglong)completed), Qt::DisplayRole); QStandardItem *i3 = new QStandardItem(); i3->setData(QVariant((qlonglong)completed), Qt::DisplayRole);
QStandardItem *i4 = new QStandardItem(); i4->setData(QVariant((double)dlspeed), Qt::DisplayRole); QStandardItem *i4 = new QStandardItem(); i4->setData(QVariant((double)dlspeed), Qt::DisplayRole);
QStandardItem *i5 = new QStandardItem(); i5->setData(QVariant((double)progress), Qt::DisplayRole); QStandardItem *i5 = new QStandardItem(); i5->setData(QVariant::fromValue(pinfo), Qt::DisplayRole);
QStandardItem *i6 = new QStandardItem(); i6->setData(QVariant((QString)sources), Qt::DisplayRole); QStandardItem *i6 = new QStandardItem(); i6->setData(QVariant((QString)sources), Qt::DisplayRole);
QStandardItem *i7 = new QStandardItem(); i7->setData(QVariant((QString)status), Qt::DisplayRole); QStandardItem *i7 = new QStandardItem(); i7->setData(QVariant((QString)status), Qt::DisplayRole);
QStandardItem *i8 = new QStandardItem(); i8->setData(QVariant((QString)tr("")), Qt::DisplayRole); // blank field for priority QStandardItem *i8 = new QStandardItem(); i8->setData(QVariant((QString)tr("")), Qt::DisplayRole); // blank field for priority
@ -522,7 +550,7 @@ bool TransfersDialog::addPeerToItem(int row, QString symbol, QString name, QStri
} }
int TransfersDialog::addUploadItem(QString symbol, QString name, QString coreID, qlonglong fileSize, double progress, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining) int TransfersDialog::addUploadItem(QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, QString source, QString status, qlonglong completed, qlonglong remaining)
{ {
int row; int row;
QString sl; QString sl;
@ -535,9 +563,9 @@ int TransfersDialog::addUploadItem(QString symbol, QString name, QString coreID,
ULListModel->setData(ULListModel->index(row, USIZE), QVariant((qlonglong)fileSize)); ULListModel->setData(ULListModel->index(row, USIZE), QVariant((qlonglong)fileSize));
ULListModel->setData(ULListModel->index(row, UTRANSFERRED), QVariant((qlonglong)completed)); ULListModel->setData(ULListModel->index(row, UTRANSFERRED), QVariant((qlonglong)completed));
ULListModel->setData(ULListModel->index(row, ULSPEED), QVariant((double)dlspeed)); ULListModel->setData(ULListModel->index(row, ULSPEED), QVariant((double)dlspeed));
ULListModel->setData(ULListModel->index(row, UPROGRESS), QVariant((double)progress)); ULListModel->setData(ULListModel->index(row, UPROGRESS), QVariant::fromValue(pinfo));
ULListModel->setData(ULListModel->index(row, USTATUS), QVariant((QString)status)); ULListModel->setData(ULListModel->index(row, USTATUS), QVariant((QString)status));
ULListModel->setData(ULListModel->index(row, USERNAME), QVariant((QString)sources)); ULListModel->setData(ULListModel->index(row, USERNAME), QVariant((QString)source));
return row; return row;
} }
@ -568,9 +596,9 @@ void TransfersDialog::editItem(int row, int column, QVariant data)
case SIZE: case SIZE:
DLListModel->setData(DLListModel->index(row, SIZE), data); DLListModel->setData(DLListModel->index(row, SIZE), data);
break; break;
case PROGRESS: // case PROGRESS:
DLListModel->setData(DLListModel->index(row, PROGRESS), data); // DLListModel->setData(DLListModel->index(row, PROGRESS), data);
break; // break;
case DLSPEED: case DLSPEED:
DLListModel->setData(DLListModel->index(row, DLSPEED), data); DLListModel->setData(DLListModel->index(row, DLSPEED), data);
break; break;
@ -670,7 +698,8 @@ void TransfersDialog::insertTransfers()
uint32_t ulCount = 0; uint32_t ulCount = 0;
std::list<std::string>::iterator it; std::list<std::string>::iterator it;
for (it = downHashes.begin(); it != downHashes.end(); it++) { for (it = downHashes.begin(); it != downHashes.end(); it++)
{
FileInfo info; FileInfo info;
if (!rsFiles->FileDetails(*it, RS_FILE_HINTS_DOWNLOAD, info)) continue; if (!rsFiles->FileDetails(*it, RS_FILE_HINTS_DOWNLOAD, info)) continue;
//if file transfer is a cache file index file, don't show it //if file transfer is a cache file index file, don't show it
@ -713,7 +742,21 @@ void TransfersDialog::insertTransfers()
completed = info.transfered; completed = info.transfered;
remaining = (info.size - info.transfered) / (info.tfRate * 1024.0); remaining = (info.size - info.transfered) / (info.tfRate * 1024.0);
int addedRow = addItem(symbol, name, coreId, fileSize, progress, dlspeed, sources, status, priority, completed, remaining); FileChunksInfo fcinfo ;
if(!rsFiles->FileDownloadChunksDetails(*it,fcinfo))
continue ;
FileProgressInfo pinfo ;
pinfo.cmap = fcinfo.chunks ;
pinfo.progress = completed*100.0/info.size ;
// std::cerr << "Converting fcinfo to compressed chunk map. Chunks=" << fcinfo.chunks.size() << std::endl ;
// std::cerr << "map data = " ;
// for(uint k=0;k<cmap._map.size();++k)
// std::cout << (void*)cmap._map[k] ;
// std::cout << std::endl ;
int addedRow = addItem(symbol, name, coreId, fileSize, pinfo, dlspeed, sources, status, priority, completed, remaining);
/* if found in selectedIds -> select again */ /* if found in selectedIds -> select again */
if (selectedIds.end() != std::find(selectedIds.begin(), selectedIds.end(), info.hash)) { if (selectedIds.end() != std::find(selectedIds.begin(), selectedIds.end(), info.hash)) {
@ -777,7 +820,11 @@ void TransfersDialog::insertTransfers()
completed = info.transfered; completed = info.transfered;
remaining = (info.size - info.transfered) / (pit->tfRate * 1024.0); remaining = (info.size - info.transfered) / (pit->tfRate * 1024.0);
if (!addPeerToItem(addedRow, symbol, name, coreId, fileSize, progress, dlspeed, sources, status, completed, remaining)) FileProgressInfo pinfo ;
pinfo.cmap = fcinfo.compressed_peer_availability_maps[pit->peerId] ;
pinfo.progress = 0.0 ; // we don't display completion for sources.
if (!addPeerToItem(addedRow, symbol, name, coreId, fileSize, pinfo, dlspeed, sources, status, completed, remaining))
continue; continue;
/* if peers found in selectedIds, select again */ /* if peers found in selectedIds, select again */
@ -826,7 +873,10 @@ void TransfersDialog::insertTransfers()
break; break;
} }
addItem("", name, coreId, fileSize, progress, dlspeed, sources, status, priority, completed, remaining); FileProgressInfo pinfo ;
pinfo.progress = 0.0 ;
addItem("", name, coreId, fileSize, pinfo, dlspeed, sources, status, priority, completed, remaining);
/* if found in selectedIds -> select again */ /* if found in selectedIds -> select again */
if (selectedIds.end() != std::find(selectedIds.begin(), selectedIds.end(), dit->hash)) { if (selectedIds.end() != std::find(selectedIds.begin(), selectedIds.end(), dit->hash)) {
@ -890,14 +940,36 @@ void TransfersDialog::insertTransfers()
// status = "Complete"; // status = "Complete";
// } // }
FileProgressInfo pinfo ;
if(!rsFiles->FileUploadChunksDetails(*it,pit->peerId,pinfo.cmap) )
continue ;
dlspeed = pit->tfRate * 1024.0; dlspeed = pit->tfRate * 1024.0;
fileSize = info.size; fileSize = info.size;
completed = info.transfered; completed = info.transfered;
progress = info.transfered * 100.0 / info.size; progress = info.transfered * 100.0 / info.size;
remaining = (info.size - info.transfered) / (info.tfRate * 1024.0); remaining = (info.size - info.transfered) / (info.tfRate * 1024.0);
addUploadItem(symbol, name, coreId, fileSize, progress, // Estimate the completion. We need something more accurate, meaning that we need to
dlspeed, sources, status, completed, remaining); // transmit the completion info.
//
uint32_t chunk_size = 1024*1024 ;
uint32_t nb_chunks = (uint32_t)(info.size / (uint64_t)(chunk_size) ) ;
if((info.size % (uint64_t)chunk_size) != 0)
++nb_chunks ;
uint32_t filled_chunks = pinfo.cmap.filledChunks(nb_chunks) ;
if(filled_chunks > 1)
{
pinfo.progress = filled_chunks*100.0/nb_chunks ;
completed = std::min(info.size,((uint64_t)filled_chunks)*chunk_size) ;
}
else
pinfo.progress = progress ;
addUploadItem(symbol, name, coreId, fileSize, pinfo, dlspeed, sources, status, completed, remaining);
ulCount++; ulCount++;
} }
@ -935,8 +1007,11 @@ void TransfersDialog::insertTransfers()
progress = info.transfered * 100.0 / info.size; progress = info.transfered * 100.0 / info.size;
remaining = (info.size - info.transfered) / (info.tfRate * 1024.0); remaining = (info.size - info.transfered) / (info.tfRate * 1024.0);
addUploadItem(symbol, name, coreId, fileSize, progress, FileProgressInfo pinfo ;
dlspeed, sources, status, completed, remaining); pinfo.progress = progress ;
pinfo.cmap = CompressedChunkMap() ;
addUploadItem(symbol, name, coreId, fileSize, pinfo, dlspeed, sources, status, completed, remaining);
ulCount++; ulCount++;
} }
} }
@ -1411,6 +1486,8 @@ void TransfersDialog::showFileDetails()
std::string file_hash ; std::string file_hash ;
int nb_select = 0 ; int nb_select = 0 ;
std::cout << "new selection " << std::endl ;
for(int i = 0; i <= DLListModel->rowCount(); i++) for(int i = 0; i <= DLListModel->rowCount(); i++)
if(selection->isRowSelected(i, QModelIndex())) if(selection->isRowSelected(i, QModelIndex()))
{ {
@ -1418,15 +1495,19 @@ void TransfersDialog::showFileDetails()
++nb_select ; ++nb_select ;
} }
if(nb_select != 1) if(nb_select != 1)
return ; dynamic_cast<FileTransferInfoWidget*>(ui.fileTransferInfoWidget->widget())->setFileHash("") ;
else
dynamic_cast<FileTransferInfoWidget*>(ui.fileTransferInfoWidget->widget())->setFileHash(file_hash) ;
dynamic_cast<FileTransferInfoWidget*>(ui.fileTransferInfoWidget->widget())->setFileHash(file_hash) ; std::cout << "calling update " << std::endl ;
dynamic_cast<FileTransferInfoWidget*>(ui.fileTransferInfoWidget->widget())->updateDisplay() ; dynamic_cast<FileTransferInfoWidget*>(ui.fileTransferInfoWidget->widget())->updateDisplay() ;
std::cout << "done" << std::endl ;
} }
double TransfersDialog::getProgress(int row, QStandardItemModel *model) double TransfersDialog::getProgress(int row, QStandardItemModel *model)
{ {
return model->data(model->index(row, PROGRESS), Qt::DisplayRole).toDouble(); // return model->data(model->index(row, PROGRESS), Qt::DisplayRole).toDouble();
return 0.0 ;
} }
double TransfersDialog::getSpeed(int row, QStandardItemModel *model) double TransfersDialog::getSpeed(int row, QStandardItemModel *model)

7
retroshare-gui/src/gui/TransfersDialog.h
View File

@ -33,6 +33,7 @@
#include <rsiface/rstypes.h> #include <rsiface/rstypes.h>
#include "mainpage.h" #include "mainpage.h"
#include "RsAutoUpdatePage.h" #include "RsAutoUpdatePage.h"
#include "xprogressbar.h"
#include "ui_TransfersDialog.h" #include "ui_TransfersDialog.h"
@ -165,11 +166,11 @@ class TransfersDialog : public RsAutoUpdatePage
Ui::TransfersDialog ui; Ui::TransfersDialog ui;
public slots: public slots:
int addItem(QString symbol, QString name, QString coreID, qlonglong size, double progress, double dlspeed, QString sources, QString status, QString priority, qlonglong completed, qlonglong remaining); int addItem(QString symbol, QString name, QString coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, QString priority, qlonglong completed, qlonglong remaining);
bool addPeerToItem(int row, QString symbol, QString name, QString coreID, qlonglong fileSize, double progress, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining); bool addPeerToItem(int row, QString symbol, QString name, QString coreID, qlonglong fileSize, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining);
void delItem(int row); void delItem(int row);
int addUploadItem(QString symbol, QString name, QString coreID, qlonglong size, double progress, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining); int addUploadItem(QString symbol, QString name, QString coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, QString sources, QString status, qlonglong completed, qlonglong remaining);
void delUploadItem(int row); void delUploadItem(int row);
void editItem(int row, int column, QVariant data); void editItem(int row, int column, QVariant data);

38
retroshare-gui/src/gui/ULListDelegate.cpp
View File

@ -19,6 +19,7 @@
* Boston, MA 02110-1301, USA. * Boston, MA 02110-1301, USA.
****************************************************************/ ****************************************************************/
#include <rsiface/rstypes.h>
#include <QModelIndex> #include <QModelIndex>
#include <QPainter> #include <QPainter>
#include <QStyleOptionProgressBarV2> #include <QStyleOptionProgressBarV2>
@ -28,6 +29,8 @@
#include "ULListDelegate.h" #include "ULListDelegate.h"
Q_DECLARE_METATYPE(FileProgressInfo)
ULListDelegate::ULListDelegate(QObject *parent) : QAbstractItemDelegate(parent) ULListDelegate::ULListDelegate(QObject *parent) : QAbstractItemDelegate(parent)
{ {
; ;
@ -123,25 +126,22 @@ void ULListDelegate::paint(QPainter * painter, const QStyleOptionViewItem & opti
painter->drawText(option.rect, Qt::AlignRight, temp); painter->drawText(option.rect, Qt::AlignRight, temp);
break; break;
case UPROGRESS: case UPROGRESS:
{ {
progress = index.data().toDouble(); FileProgressInfo pinfo = index.data().value<FileProgressInfo>() ;
// create a xProgressBar
xProgressBar progressBar(option.rect, painter );// the 3rd param is the color schema (0 is the default value) // create a xProgressBar
xProgressBar progressBar(pinfo.cmap,option.rect,painter,0);// the 3rd param is the color schema (0 is the default value)
QString ext = QFileInfo(QString::fromStdString(index.sibling(index.row(), UNAME).data().toString().toStdString())).suffix();;
if (ext == "rsfc" || ext == "rsrl" || ext == "dist" || ext == "rsfb") QString ext = QFileInfo(QString::fromStdString(index.sibling(index.row(), UNAME).data().toString().toStdString())).suffix();;
{ if (ext == "rsfc" || ext == "rsrl" || ext == "dist" || ext == "rsfb")
progressBar.setColorSchema( 9); progressBar.setColorSchema( 9);
} else
else progressBar.setColorSchema( 8);
{
progressBar.setColorSchema( 8); progressBar.setDisplayText(true); // should display % text?
} progressBar.setValue(pinfo.progress); // set the progress value
progressBar.setVerticalSpan(1);
progressBar.setDisplayText(false); // should display % text? progressBar.paint(); // paint the progress bar
progressBar.setValue(progress); // set the progress value
progressBar.setVerticalSpan(1);
progressBar.paint(); // paint the progress bar
} }
painter->drawText(option.rect, Qt::AlignCenter, newopt.text); painter->drawText(option.rect, Qt::AlignCenter, newopt.text);
break; break;

46
retroshare-gui/src/gui/xprogressbar.cpp
View File

@ -24,9 +24,12 @@
* Boston, MA 02110-1301, USA. * Boston, MA 02110-1301, USA.
****************************************************************/ ****************************************************************/
#include <math.h>
#include <rsiface/rstypes.h>
#include "xprogressbar.h" #include "xprogressbar.h"
xProgressBar::xProgressBar(QRect rect, QPainter *painter, int schemaIndex) xProgressBar::xProgressBar(const CompressedChunkMap& cmap,QRect rect, QPainter *painter, int schemaIndex)
: _cmap(cmap)
{ {
// assign internal data // assign internal data
this->schemaIndex = schemaIndex; this->schemaIndex = schemaIndex;
@ -39,7 +42,7 @@ xProgressBar::xProgressBar(QRect rect, QPainter *painter, int schemaIndex)
vSpan = 0; vSpan = 0;
hSpan = 0; hSpan = 0;
// text color // text color
textColor = QColor("white"); textColor = QColor("black");
} }
void xProgressBar::setColor() void xProgressBar::setColor()
@ -187,14 +190,41 @@ void xProgressBar::paint()
linearGrad.setColorAt(1.00, gradColor1); linearGrad.setColorAt(1.00, gradColor1);
painter->setPen(gradBorderColor); painter->setPen(gradBorderColor);
// calculate progress value int width = static_cast<int>(rect.width()-1-2*hSpan) ;
int preWidth = static_cast<int>((rect.width() - 1 - hSpan)*(progressValue/100));
int progressWidth = rect.width() - preWidth;
if (progressWidth == rect.width() - hSpan) return;
// paint the progress
painter->setBrush(linearGrad); painter->setBrush(linearGrad);
painter->drawRect(rect.x() + hSpan, rect.y() + vSpan, rect.width() - progressWidth - hSpan, rect.height() - 1 - vSpan * 2);
uint32_t ss = _cmap._map.size()*32 ;
if(ss > 1) // for small files we use a more progressive display
for(int i=0;i<ss;++i)
{
int j=0 ;
while(i+j<ss && _cmap[i+j])
++j ;
if(j>0)
{
float o = std::min(1.0f,j/(float)ss*width) ;
painter->setOpacity(o) ;
painter->drawRect(rect.x() + hSpan+(int)rint(i*width/(float)ss), rect.y() + vSpan, (int)ceil(j*width/(float)ss), rect.height() - 1 - vSpan * 2);
}
i += j ;
}
else
{
// calculate progress value
int preWidth = static_cast<int>((rect.width() - 1 - hSpan)*(progressValue/100));
int progressWidth = rect.width() - preWidth;
if (progressWidth == rect.width() - hSpan) return;
// paint the progress
painter->setBrush(linearGrad);
painter->drawRect(rect.x() + hSpan, rect.y() + vSpan, rect.width() - progressWidth - hSpan, rect.height() - 1 - vSpan * 2);
}
painter->setOpacity(1.0f) ;
// paint text? // paint text?
if (displayText) if (displayText)

15
retroshare-gui/src/gui/xprogressbar.h
View File

@ -27,11 +27,21 @@
#ifndef XPROGRESSBAR_H #ifndef XPROGRESSBAR_H
#define XPROGRESSBAR_H #define XPROGRESSBAR_H
// //
#include <stdint.h>
#include <QRect> #include <QRect>
#include <QColor> #include <QColor>
#include <QPainter> #include <QPainter>
#include <QLinearGradient> #include <QLinearGradient>
#include <QLocale> #include <QLocale>
#include <rsiface/rstypes.h>
class FileProgressInfo
{
public:
CompressedChunkMap cmap ;
float progress ;
};
// //
class xProgressBar : public QObject class xProgressBar : public QObject
{ {
@ -56,8 +66,11 @@ Q_OBJECT
QColor gradColor2; QColor gradColor2;
// configure the color // configure the color
void setColor(); void setColor();
const CompressedChunkMap& _cmap ;
public: public:
xProgressBar(QRect rect, QPainter *painter, int schemaIndex = 0); xProgressBar(const CompressedChunkMap& cmap,QRect rect, QPainter *painter, int schemaIndex = 0);
void paint(); void paint();
void setColorSchema(const int value); void setColorSchema(const int value);
void setValue(const float value); void setValue(const float value);