Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-06-01 20:52:05 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

- Implemented chunk-based file transfer from partial sources. This in particular means:

Browse source 
- exchange of chunk availability maps from different peers
    - correct handling of what is available to which source before asking the data
    - correct display of chunks in the progress bars
    - generalised the use of compressed chunk maps
    - removed the size parameters from the hash search functions
   
- In addition:
    - suppressed a number of per-value transfers of std::string
    - improved the FileTransferInfo Widget, to show some additional info

Still to be done:
    - chunk map exchange for non anonymous traffic (easy)
    - improve accuracy of completion for uploads (for now it's a integer number of chunks)
    - check compilation on windows




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1993 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2010-01-11 16:00:42 +00:00
parent add5d45eeb
commit cfaaec31c7
36 changed files with 1247 additions and 573 deletions
Download patch file Download diff file Expand all files Collapse all files

206
libretroshare/src/ft/ftchunkmap.cc
View file

@ -1,8 +1,11 @@
#include <math.h>
#include <assert.h>
#include <stdlib.h>
#include <rsiface/rspeers.h>
#include "ftchunkmap.h"
static const uint32_t SOURCE_CHUNK_MAP_UPDATE_PERIOD = 60 ; //! TTL for chunkmap info
std::ostream& operator<<(std::ostream& o,const ftChunk& c)
{
return o << "\tChunk [" << c.offset << "] size: " << c.size << " ChunkId: " << c.id << " Age: " << time(NULL) - c.ts ;
@ -28,26 +31,6 @@ void Chunk::getSlice(uint32_t size_hint,ftChunk& chunk)
_offset += chunk.size ;
}
//uint32_t Chunk::dataReceived(const ftChunk::ChunkId cid)
//{
//#ifdef DEBUG_FTCHUNK
// std::cerr << "*** Chunk::dataReceived: slice " << cid << " finished" << std::endl ;
//#endif
// std::map<ftChunk::ChunkId,uint32_t>::iterator it( _slices_to_download.find(cid) ) ;
//
// if(it == _slices_to_download.end())
// {
// std::cerr << "!!! Chunk::dataReceived: could not find chunk " << cid << ": probably a fatal error" << std::endl ;
// return 0 ;
// }
// else
// {
// uint32_t n = it->second ;
// _slices_to_download.erase(it) ;
// return n ;
// }
//}
ChunkMap::ChunkMap(uint64_t s)
:_file_size(s),_chunk_size(1024*1024) // 1MB chunks
{
@ -67,34 +50,46 @@ ChunkMap::ChunkMap(uint64_t s)
#endif
}
ChunkMap::ChunkMap(uint64_t file_size,
const std::vector<uint32_t>& map,
uint32_t chunk_size,
uint32_t chunk_number,
FileChunksInfo::ChunkStrategy strategy)
:_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy)
void ChunkMap::setAvailabilityMap(const CompressedChunkMap& map)
{
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ;
#endif
_map.clear() ;
_map.resize(chunk_number) ;
_total_downloaded = 0 ;
for(uint32_t i=0;i<_map.size();++i)
{
uint32_t j = i & 31 ; // i%32
uint32_t k = i >> 5 ; // i/32
_map[i] = ( (map[k] & (1<<j)) > 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ;
if(_map[i] == FileChunksInfo::CHUNK_DONE)
if(map[i] > 0)
{
_map[i] = FileChunksInfo::CHUNK_DONE ;
_total_downloaded += _chunk_size ;
}
}
else
_map[i] = FileChunksInfo::CHUNK_OUTSTANDING ;
}
//ChunkMap::ChunkMap(uint64_t file_size,
// const std::vector<uint32_t>& map,
// uint32_t chunk_size,
// uint32_t chunk_number,
// FileChunksInfo::ChunkStrategy strategy)
//
// :_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy)
//{
//#ifdef DEBUG_FTCHUNK
// std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ;
//#endif
//
// _map.clear() ;
// _map.resize(chunk_number) ;
// _total_downloaded = 0 ;
//
// for(uint32_t i=0;i<_map.size();++i)
// {
// uint32_t j = i & 31 ; // i%32
// uint32_t k = i >> 5 ; // i/32
//
// _map[i] = ( (map[k] & (1<<j)) > 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ;
//
// if(_map[i] == FileChunksInfo::CHUNK_DONE)
// _total_downloaded += _chunk_size ;
// }
//}
void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
{
@ -151,7 +146,7 @@ void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
// - chunks pushed when new chunks are needed
// - chunks removed when completely downloaded
//
bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk)
bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk,bool& source_chunk_map_needed)
{
#ifdef DEBUG_FTCHUNK
std::cerr << "*** ChunkMap::getDataChunk: size_hint = " << size_hint << std::endl ;
@ -168,10 +163,10 @@ bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChun
switch(_strategy)
{
case FileChunksInfo::CHUNK_STRATEGY_STREAMING: c = getAvailableChunk(0,peer_id) ; // very bold!!
case FileChunksInfo::CHUNK_STRATEGY_STREAMING: c = getAvailableChunk(0,peer_id,source_chunk_map_needed) ; // very bold!!
break ;
case FileChunksInfo::CHUNK_STRATEGY_RANDOM: c = getAvailableChunk(rand()%_map.size(),peer_id) ;
case FileChunksInfo::CHUNK_STRATEGY_RANDOM: c = getAvailableChunk(rand()%_map.size(),peer_id,source_chunk_map_needed) ;
break ;
default:
#ifdef DEBUG_FTCHUNK
@ -212,23 +207,51 @@ bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChun
return true ;
}
void ChunkMap::setPeerAvailabilityMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& compressed_peer_map)
bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const
{
uint32_t chunk_number_start = offset/(uint64_t)_chunk_size ;
uint32_t chunk_number_end = (offset+(uint64_t)chunk_size)/(uint64_t)_chunk_size ;
if((offset+(uint64_t)chunk_size) % (uint64_t)_chunk_size == 0)
--chunk_number_end ;
// It's possible that chunk_number_start==chunk_number_end+1, but for this we need to have
// chunk_size=0, and offset%_chunk_size=0, so the response "true" is still valid.
//
for(uint32_t i=chunk_number_start;i!=chunk_number_end;++i)
if(_map[i] != FileChunksInfo::CHUNK_DONE)
return false ;
return true ;
}
void ChunkMap::setPeerAvailabilityMap(const std::string& peer_id,const CompressedChunkMap& cmap)
{
#ifdef DEBUG_FTCHUNK
std::cout << "ChunkMap::Receiving new availability map for peer " << peer_id << std::endl ;
#endif
// Check that the parameters are the same. Otherwise we should convert the info into the local format.
// If all RS instances have the same policy for deciding the sizes of chunks, this should not happen.
if(chunk_size != _chunk_size || nb_chunks != _map.size())
if(cmap._map.size() != _map.size()/32+(_map.size()%32 != 0))
{
std::cerr << "ChunkMap::setPeerAvailabilityMap: chunk size / number of chunks is not correct. Dropping the info." << std::endl ;
std::cerr << "ChunkMap::setPeerAvailabilityMap: chunk size / number of chunks is not correct. Dropping the info. cmap.size()=" << cmap._map.size() << ", _map/32+0/1 = " << _map.size()/32+(_map.size()%32 != 0) << std::endl ;
return ;
}
// sets the map.
//
_peers_chunks_availability[peer_id] = compressed_peer_map ;
SourceChunksInfo& mi(_peers_chunks_availability[peer_id]) ;
mi.cmap = cmap ;
mi.TS = time(NULL) ;
mi.is_full = true ;
// Checks wether the map is full of not.
//
for(uint i=0;i<_map.size();++i)
if(!cmap[i])
{
mi.is_full = false ;
break ;
}
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap::setPeerAvailabilityMap: Setting chunk availability info for peer " << peer_id << std::endl ;
@ -243,32 +266,54 @@ uint32_t ChunkMap::sizeOfChunk(uint32_t cid) const
return _chunk_size ;
}
uint32_t ChunkMap::getAvailableChunk(uint32_t start_location,const std::string& peer_id)
uint32_t ChunkMap::getAvailableChunk(uint32_t start_location,const std::string& peer_id,bool& map_is_too_old)
{
// Very bold algorithm: checks for 1st availabe chunk for this peer starting
// from the given start location.
std::map<std::string,std::vector<uint32_t> >::const_iterator it(_peers_chunks_availability.find(peer_id)) ;
std::map<std::string,SourceChunksInfo>::iterator it(_peers_chunks_availability.find(peer_id)) ;
// Do we have records for this file source ?
//
if(it == _peers_chunks_availability.end())
{
#ifdef DEBUG_FTCHUNK
std::cout << "No chunk map for peer " << peer_id << ": supposing full data." << std::endl ;
#endif
std::vector<uint32_t>& pchunks(_peers_chunks_availability[peer_id]) ;
SourceChunksInfo& pchunks(_peers_chunks_availability[peer_id]) ;
pchunks.resize( (_map.size() >> 5)+!!(_map.size() & 0x11111),~(uint32_t)0 ) ;
// Ok, we don't have the info, so two cases:
// - we are the actual source, so we can safely init the map to a full map
// - we are not the source, so we init it with an empty map, and set the time stamp to 0.
//
if(peer_id == rsPeers->getOwnId())
{
pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),~(uint32_t)0 ) ;
pchunks.TS = 0 ;
pchunks.is_full = true ;
}
else
{
pchunks.cmap._map.resize( CompressedChunkMap::getCompressedSize(_map.size()),0 ) ;
pchunks.TS = 0 ;
pchunks.is_full = false ;
}
it = _peers_chunks_availability.find(peer_id) ;
}
const std::vector<uint32_t>& peer_chunks(it->second) ;
SourceChunksInfo& peer_chunks(it->second) ;
// If the info is too old, we ask for a new one. When the map is full, we ask 10 times less, as it's probably not
// useful to get a new map that will also be full, but because we need to be careful not to mislead information,
// we still keep asking.
//
time_t now = time(NULL) ;
map_is_too_old = (int)now - (int)peer_chunks.TS > (int)SOURCE_CHUNK_MAP_UPDATE_PERIOD*(1+9*peer_chunks.is_full) ;
// We will re-ask but not now seconds.
peer_chunks.TS = now ;
for(unsigned int i=0;i<_map.size();++i)
{
uint32_t j = (start_location+i)%(int)_map.size() ; // index of the chunk
if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && COMPRESSED_MAP_READ(peer_chunks,j))
if(_map[j] == FileChunksInfo::CHUNK_OUTSTANDING && peer_chunks.cmap[j])
{
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap::getAvailableChunk: returning chunk " << j << " for peer " << peer_id << std::endl;
@ -296,35 +341,30 @@ void ChunkMap::getChunksInfo(FileChunksInfo& info) const
info.compressed_peer_availability_maps.clear() ;
for(std::map<std::string,std::vector<uint32_t> >::const_iterator it(_peers_chunks_availability.begin());it!= _peers_chunks_availability.end();++it)
info.compressed_peer_availability_maps.push_back(std::pair<std::string,std::vector<uint32_t> >(it->first,it->second)) ;
for(std::map<std::string,SourceChunksInfo>::const_iterator it(_peers_chunks_availability.begin());it!=_peers_chunks_availability.end();++it)
info.compressed_peer_availability_maps[it->first] = it->second.cmap ;
}
void ChunkMap::buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& strategy) const
void ChunkMap::getAvailabilityMap(CompressedChunkMap& compressed_map) const
{
chunk_size = _chunk_size ;
chunk_number = _map.size() ;
strategy = _strategy ;
compressed_map = CompressedChunkMap(_map) ;
map.clear() ;
map.reserve((chunk_number >> 5)+1) ;
uint32_t r=0 ;
for(uint32_t i=0;i<_map.size();++i)
{
uint32_t j = i & 31 ;
r |= (_map[i]==FileChunksInfo::CHUNK_DONE)?(1<<j):0 ;
if(j==31 || i==_map.size()-1)
{
map.push_back(r);
r=0 ;
}
}
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap:: built availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << ", strategy=" << _strategy << std::endl ;
std::cerr << "ChunkMap:: retrieved availability map of size " << _map.size() << ", chunk_size=" << _chunk_size << std::endl ;
#endif
}
void ChunkMap::buildPlainMap(uint64_t size, CompressedChunkMap& map)
{
uint32_t chunk_size(1024*1024) ; // 1MB chunks
uint64_t n = size/(uint64_t)chunk_size ;
if(size % (uint64_t)chunk_size != 0)
++n ;
map = CompressedChunkMap(n,~uint32_t(0)) ;
}

98
libretroshare/src/ft/ftchunkmap.h
View file

@ -73,80 +73,96 @@ class ChunkDownloadInfo
uint32_t _remains ;
};
class SourceChunksInfo
{
public:
CompressedChunkMap cmap ; //! map of what the peer has/doens't have
time_t TS ; //! last update time for this info
bool is_full ; //! is the map full ? In such a case, re-asking for it is unnecessary.
};
class ChunkMap
{
public:
typedef uint32_t ChunkNumber ;
// Constructor. Decides what will be the size of chunks and how many there will be.
/// Constructor. Decides what will be the size of chunks and how many there will be.
ChunkMap(uint64_t file_size) ;
// constructor from saved map info
/// constructor from saved map info
ChunkMap(uint64_t file_size,const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
// destructor
/// destructor
virtual ~ChunkMap() {}
// Returns an slice of data to be asked to the peer within a chunk.
// If a chunk is already been downloaded by this peer, take a slice at
// the beginning of this chunk, or at least where it starts.
// If not, randomly/streamly select a new chunk depending on the strategy.
// adds an entry in the chunk_ids map, and sets up 1 interval for it.
// the chunk should be available from the designated peer.
/// Returns an slice of data to be asked to the peer within a chunk.
/// If a chunk is already been downloaded by this peer, take a slice at
/// the beginning of this chunk, or at least where it starts.
/// If not, randomly/streamly select a new chunk depending on the strategy.
/// adds an entry in the chunk_ids map, and sets up 1 interval for it.
/// the chunk should be available from the designated peer.
virtual bool getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk) ;
virtual bool getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChunk& chunk,bool& source_chunk_map_needed) ;
// Notify received a slice of data. This needs to
// - carve in the map of chunks what is received, what is not.
// - tell which chunks are finished. For this, each interval must know what chunk number it has been attributed
// when the interval is split in the middle, the number of intervals for the chunk is increased. If the interval is
// completely covered by the data, the interval number is decreased.
/// Notify received a slice of data. This needs to
/// - carve in the map of chunks what is received, what is not.
/// - tell which chunks are finished. For this, each interval must know what chunk number it has been attributed
/// when the interval is split in the middle, the number of intervals for the chunk is increased. If the interval is
/// completely covered by the data, the interval number is decreased.
virtual void dataReceived(const ftChunk::ChunkId& c_id) ;
// Decides how chunks are selected.
// STREAMING: the 1st chunk is always returned
// RANDOM: the beginning of a random interval is selected first. If two few intervals
// exist, the largest one is randomly split into two.
/// Decides how chunks are selected.
/// STREAMING: the 1st chunk is always returned
/// RANDOM: the beginning of a random interval is selected first. If two few intervals
/// exist, the largest one is randomly split into two.
void setStrategy(FileChunksInfo::ChunkStrategy s) { _strategy = s ; }
FileChunksInfo::ChunkStrategy getStrategy() const { return _strategy ; }
// Properly fills an vector of fixed size chunks with availability or download state.
// chunks is given with the proper number of chunks and we have to adapt to it. This can be used
// to display square chunks in the gui or display a blue bar of availability by collapsing info from all peers.
/// Properly fills an vector of fixed size chunks with availability or download state.
/// chunks is given with the proper number of chunks and we have to adapt to it. This can be used
/// to display square chunks in the gui or display a blue bar of availability by collapsing info from all peers.
/// The set method is not virtual because it has no reason to exist in the parent ftFileProvider
void buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& s) const ;
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
virtual void getAvailabilityMap(CompressedChunkMap& cmap) const ;
void setAvailabilityMap(const CompressedChunkMap& cmap) ;
// Updates the peer's availablility map
/// This function fills in a plain map for a file of the given size. This
/// is used to ensure that the chunk size will be consistent with the rest
/// of the code.
//
void setPeerAvailabilityMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& peer_map) ;
static void buildPlainMap(uint64_t size,CompressedChunkMap& map) ;
// Returns the total size of downloaded data in the file.
/// This function is used by the parent ftFileProvider to know whether the chunk can be sent or not.
bool isChunkAvailable(uint64_t offset, uint32_t chunk_size) const ;
/// Updates the peer's availablility map
//
void setPeerAvailabilityMap(const std::string& peer_id,const CompressedChunkMap& peer_map) ;
/// Returns the total size of downloaded data in the file.
uint64_t getTotalReceived() const { return _total_downloaded ; }
void getChunksInfo(FileChunksInfo& info) const ;
protected:
// handles what size the last chunk has.
/// handles what size the last chunk has.
uint32_t sizeOfChunk(uint32_t chunk_number) const ;
// Returns the first chunk available starting from start_location for this peer_id.
/// Returns the first chunk available starting from start_location for this peer_id.
//
uint32_t getAvailableChunk(uint32_t start_location,const std::string& peer_id) ;
uint32_t getAvailableChunk(uint32_t start_location,const std::string& peer_id,bool& chunk_map_too_old) ;
private:
uint64_t _file_size ; // total size of the file in bytes.
uint32_t _chunk_size ; // Size of chunks. Common to all chunks.
FileChunksInfo::ChunkStrategy _strategy ; // how do we allocate new chunks
std::map<std::string,Chunk> _active_chunks_feed ; // vector of chunks being downloaded. Exactly one chunk per peer id.
std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; // list of (slice id,slice size)
std::vector<FileChunksInfo::ChunkState> _map ; // vector of chunk state over the whole file
std::map<std::string,std::vector<uint32_t> > _peers_chunks_availability ; // what does each source peer have, stored in compressed format.
uint64_t _total_downloaded ;
uint64_t _file_size ; //! total size of the file in bytes.
uint32_t _chunk_size ; //! Size of chunks. Common to all chunks.
FileChunksInfo::ChunkStrategy _strategy ; //! how do we allocate new chunks
std::map<std::string,Chunk> _active_chunks_feed ; //! vector of chunks being downloaded. Exactly 1 chunk per peer.
std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; //! list of (slice id,slice size)
std::vector<FileChunksInfo::ChunkState> _map ; //! vector of chunk state over the whole file
std::map<std::string,SourceChunksInfo> _peers_chunks_availability ; //! what does each source peer have
uint64_t _total_downloaded ; //! completion for the file
};

53
libretroshare/src/ft/ftcontroller.cc
View file

@ -80,8 +80,12 @@ ftFileControl::ftFileControl(std::string fname,
}
ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, std::string configDir)
:CacheTransfer(cs), p3Config(CONFIG_TYPE_FT_CONTROL), mDataplex(dm), mFtActive(false),
mTurtle(NULL), mShareDownloadDir(true),last_save_time(0)
:CacheTransfer(cs), p3Config(CONFIG_TYPE_FT_CONTROL),
last_save_time(0),
mDataplex(dm),
mTurtle(NULL),
mFtActive(false),
mShareDownloadDir(true)
{
/* TODO */
}
@ -96,7 +100,7 @@ void ftController::setFtSearchNExtra(ftSearch *search, ftExtraList *list)
mExtraList = list;
}
bool ftController::getFileChunksDetails(const std::string& hash,FileChunksInfo& info)
bool ftController::getFileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info)
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
@ -185,7 +189,7 @@ void ftController::run()
}
time_t now = time(NULL) ;
if(now - last_save_time > SAVE_TRANSFERS_DELAY)
if((int)now - (int)last_save_time > (int)SAVE_TRANSFERS_DELAY)
{
IndicateConfigChanged() ;
last_save_time = now ;
@ -593,7 +597,10 @@ bool ftController::handleAPendingRequest()
std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
}
else
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
{
(fit->second).mCreator->setAvailabilityMap(rsft->compressed_chunk_map) ;
(fit->second).mCreator->setChunkStrategy((FileChunksInfo::ChunkStrategy)(rsft->chunk_strategy)) ;
}
delete rsft ;
mPendingChunkMaps.erase(it) ;
@ -733,10 +740,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
}
else
{
if (mSearch->search(hash, size,
RS_FILE_HINTS_LOCAL |
RS_FILE_HINTS_EXTRA |
RS_FILE_HINTS_SPEC_ONLY, info))
if (mSearch->search(hash, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_SPEC_ONLY, info))
{
/* have it already */
/* add in as completed transfer */
@ -750,10 +754,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
}
/* do a source search - for any extra sources */
if (mSearch->search(hash, size,
RS_FILE_HINTS_REMOTE |
// RS_FILE_HINTS_TURTLE |
RS_FILE_HINTS_SPEC_ONLY, info))
if (mSearch->search(hash, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info))
{
/* do something with results */
#ifdef CONTROL_DEBUG
@ -817,8 +818,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this);
/* add into maps */
ftFileControl ftfc(fname, savepath, destination,
size, hash, flags, fc, tm, callbackCode);
ftFileControl ftfc(fname, savepath, destination, size, hash, flags, fc, tm, callbackCode);
ftfc.mCreateTime = time(NULL);
#ifdef CONTROL_DEBUG
@ -1469,7 +1469,23 @@ std::list<RsItem *> ftController::saveList(bool &cleanup)
//rft->flags = fit->second.mFlags;
fit->second.mTransfer->getFileSources(rft->allPeerIds.ids);
fit->second.mCreator->storeAvailabilityMap(rft->chunk_map,rft->chunk_size,rft->chunk_number,rft->chunk_strategy) ;
// Remove turtle peers from sources, as they are not supposed to survive a reboot of RS, since they are dynamic sources.
// Otherwize, such sources are unknown from the turtle router, at restart, and never get removed.
//
for(std::list<std::string>::iterator sit(rft->allPeerIds.ids.begin());sit!=rft->allPeerIds.ids.end();)
if(mTurtle->isTurtlePeer(*sit))
{
std::list<std::string>::iterator sittmp(sit) ;
++sittmp ;
rft->allPeerIds.ids.erase(sit) ;
sit = sittmp ;
}
else
++sit ;
fit->second.mCreator->getAvailabilityMap(rft->compressed_chunk_map) ;
rft->chunk_strategy = fit->second.mCreator->getChunkStrategy() ;
saveData.push_back(rft);
}
@ -1528,7 +1544,10 @@ bool ftController::loadList(std::list<RsItem *> load)
continue ; // i.e. don't delete the item!
}
else
(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
{
(fit->second).mCreator->setAvailabilityMap(rsft->compressed_chunk_map) ;
(fit->second).mCreator->setChunkStrategy((FileChunksInfo::ChunkStrategy)(rsft->chunk_strategy)) ;
}
}
}

2
libretroshare/src/ft/ftcontroller.h
View file

@ -138,7 +138,7 @@ bool FileCancel(std::string hash);
bool FileControl(std::string hash, uint32_t flags);
bool FileClearCompleted();
bool FlagFileComplete(std::string hash);
bool getFileChunksDetails(const std::string& hash,FileChunksInfo& info);
bool getFileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info);
/* get Details of File Transfers */
bool FileDownloads(std::list<std::string> &hashs);

35
libretroshare/src/ft/ftdata.h
View file

@ -40,19 +40,23 @@
/*************** SEND INTERFACE *******************/
class CompressedChunkMap ;
class ftDataSend
{
public:
virtual ~ftDataSend() { return; }
virtual ~ftDataSend() { return; }
/* Client Send */
virtual bool sendDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
/* Client Send */
virtual bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
/* Server Send */
virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data) = 0;
/* Server Send */
virtual bool sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data) = 0;
/// Send a request for a chunk map
virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) = 0;
/// Send a chunk map
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) = 0;
};
@ -62,18 +66,19 @@ virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
class ftDataRecv
{
public:
virtual ~ftDataRecv() { return; }
virtual ~ftDataRecv() { return; }
/* Client Recv */
virtual bool recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data) = 0;
/* Client Recv */
virtual bool recvData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data) = 0;
/* Server Recv */
virtual bool recvDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
/* Server Recv */
virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
/// Send a request for a chunk map
virtual bool recvChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) = 0;
/// Send a chunk map
virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) = 0;
};
/******* Pair of Send/Recv (Only need to handle Send side) ******/

357
libretroshare/src/ft/ftdatamultiplex.cc
View file

@ -50,8 +50,10 @@ ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
return;
}
const uint32_t FT_DATA = 0x0001;
const uint32_t FT_DATA_REQ = 0x0002;
const uint32_t FT_DATA = 0x0001; // data cuhnk to be stored
const uint32_t FT_DATA_REQ = 0x0002; // data request to be treated
const uint32_t FT_CLIENT_CHUNK_MAP_REQ = 0x0003; // chunk map request to be treated by client
const uint32_t FT_SERVER_CHUNK_MAP_REQ = 0x0004; // chunk map reuqest to be treated by server
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
:mType(type), mPeerId(peerId), mHash(hash), mSize(size),
@ -179,8 +181,7 @@ bool ftDataMultiplex::FileDetails(std::string hash, uint32_t hintsflag, FileI
/*************** SEND INTERFACE (calls ftDataSend) *******************/
/* Client Send */
bool ftDataMultiplex::sendDataRequest(std::string peerId,
std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize)
bool ftDataMultiplex::sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::sendDataRequest() Client Send";
@ -190,9 +191,7 @@ bool ftDataMultiplex::sendDataRequest(std::string peerId,
}
/* Server Send */
bool ftDataMultiplex::sendData(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
bool ftDataMultiplex::sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::sendData() Server Send";
@ -205,9 +204,7 @@ bool ftDataMultiplex::sendData(std::string peerId,
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */
bool ftDataMultiplex::recvData(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
bool ftDataMultiplex::recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvData() Client Recv";
@ -215,17 +212,14 @@ bool ftDataMultiplex::recvData(std::string peerId,
#endif
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
mRequestQueue.push_back(
ftRequest(FT_DATA,peerId,hash,size,offset,chunksize,data));
mRequestQueue.push_back(ftRequest(FT_DATA,peerId,hash,size,offset,chunksize,data));
return true;
}
/* Server Recv */
bool ftDataMultiplex::recvDataRequest(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
bool ftDataMultiplex::recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvDataRequest() Server Recv";
@ -239,6 +233,24 @@ bool ftDataMultiplex::recvDataRequest(std::string peerId,
return true;
}
bool ftDataMultiplex::recvChunkMapRequest(const std::string& peerId, const std::string& hash,bool is_client)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvChunkMapRequest() Server Recv";
std::cerr << std::endl;
#endif
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
if(is_client)
mRequestQueue.push_back(ftRequest(FT_CLIENT_CHUNK_MAP_REQ,peerId,hash,0,0,0,NULL));
else
mRequestQueue.push_back(ftRequest(FT_SERVER_CHUNK_MAP_REQ,peerId,hash,0,0,0,NULL));
return true;
}
/*********** BACKGROUND THREAD OPERATIONS ***********/
bool ftDataMultiplex::workQueued()
@ -282,30 +294,44 @@ bool ftDataMultiplex::doWork()
switch(req.mType)
{
case FT_DATA:
case FT_DATA:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA";
std::cerr << std::endl;
std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA";
std::cerr << std::endl;
#endif
handleRecvData(req.mPeerId, req.mHash, req.mSize,
req.mOffset, req.mChunk, req.mData);
break;
handleRecvData(req.mPeerId, req.mHash, req.mSize, req.mOffset, req.mChunk, req.mData);
break;
case FT_DATA_REQ:
case FT_DATA_REQ:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA_REQ";
std::cerr << std::endl;
std::cerr << "ftDataMultiplex::doWork() Handling FT_DATA_REQ";
std::cerr << std::endl;
#endif
handleRecvDataRequest(req.mPeerId, req.mHash,
req.mSize, req.mOffset, req.mChunk);
break;
handleRecvDataRequest(req.mPeerId, req.mHash, req.mSize, req.mOffset, req.mChunk);
break;
default:
case FT_CLIENT_CHUNK_MAP_REQ:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN";
std::cerr << std::endl;
std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CHUNK_MAP_REQ";
std::cerr << std::endl;
#endif
break;
handleRecvClientChunkMapRequest(req.mPeerId,req.mHash) ;
break ;
case FT_SERVER_CHUNK_MAP_REQ:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Handling FT_CLIENT_CHUNK_MAP_REQ";
std::cerr << std::endl;
#endif
handleRecvServerChunkMapRequest(req.mPeerId,req.mHash) ;
break ;
default:
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::doWork() Ignoring UNKNOWN";
std::cerr << std::endl;
#endif
break;
}
}
@ -330,40 +356,137 @@ bool ftDataMultiplex::doWork()
std::cerr << "ftDataMultiplex::doWork() Handling Search Request";
std::cerr << std::endl;
#endif
handleSearchRequest(req.mPeerId, req.mHash, req.mSize,
req.mOffset, req.mChunk);
if(handleSearchRequest(req.mPeerId, req.mHash))
handleRecvDataRequest(req.mPeerId, req.mHash, req.mSize, req.mOffset, req.mChunk) ;
return true;
}
bool ftDataMultiplex::recvFileMap(const std::string& peerId, const std::string& hash, uint32_t chunk_size, uint32_t nb_chunks, const std::vector<uint32_t>& compressed_map)
// A chunk map has arrived. It can be two different situations:
// - an uploader has sent his chunk map, so we need to store it in the corresponding ftFileProvider
// - a source for a download has sent his chunk map, so we need to send it to the corresponding ftFileCreator.
//
bool ftDataMultiplex::recvChunkMap(const std::string& peerId, const std::string& hash,const CompressedChunkMap& compressed_map,bool client)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string, ftClient>::iterator it;
if (mClients.end() == (it = mClients.find(hash)))
if(client) // is the chunk map for a client, or for a server ?
{
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
if(it == mClients.end())
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvMap() ERROR: No matching Client!";
std::cerr << "ftDataMultiplex::recvChunkMap() ERROR: No matching Client for hash " << hash << " !";
std::cerr << std::endl;
#endif
/* error */
return false;
}
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::recvChunkMap() Passing map of file " << hash << ", to FT Module";
std::cerr << std::endl;
#endif
/* error */
return false;
(it->second).mCreator->setSourceMap(peerId, compressed_map);
return true ;
}
else
{
std::map<std::string, ftFileProvider *>::iterator it = mServers.find(hash) ;
if(it == mServers.end())
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMap() ERROR: No matching file Provider for hash " << hash ;
std::cerr << std::endl;
#endif
}
it->second->setClientMap(peerId, compressed_map);
return true ;
}
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvMap() Passing map to FT Module";
std::cerr << std::endl;
#endif
(it->second).mCreator->setSourceMap(peerId, chunk_size, nb_chunks,compressed_map);
return true;
return false;
}
bool ftDataMultiplex::handleRecvData(std::string peerId,
std::string hash, uint64_t size,
bool ftDataMultiplex::handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash)
{
CompressedChunkMap cmap ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string, ftClient>::iterator it = mClients.find(hash);
if(it == mClients.end())
{
// If we can't find the client, it's not a problem. Chunk maps from
// clients are not essential, as they are only used for display.
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvServerChunkMapRequest() ERROR: No matching Client for hash " << hash ;
std::cerr << ". Performing local search." << std::endl;
#endif
return false;
}
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvServerChunkMapRequest() Sending map of file " << hash << ", to peer " << peerId << std::endl;
#endif
(it->second).mCreator->getAvailabilityMap(cmap);
}
mDataSend->sendChunkMap(peerId,hash,cmap);
return true ;
}
bool ftDataMultiplex::handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash)
{
CompressedChunkMap cmap ;
std::map<std::string, ftFileProvider *>::iterator it ;
bool found = true ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
if(it == mServers.end())
found = false ;
}
if(!found)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() ERROR: No matching file Provider for hash " << hash ;
std::cerr << std::endl;
#endif
if(!handleSearchRequest(peerId,hash))
return false ;
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleRecvChunkMapReq() A new file Provider has been made up for hash " << hash ;
std::cerr << std::endl;
#endif
}
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
it = mServers.find(hash) ;
it->second->getAvailabilityMap(cmap);
}
mDataSend->sendChunkMap(peerId,hash,cmap);
return true;
}
bool ftDataMultiplex::handleRecvData(const std::string& peerId,
const std::string& hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
@ -390,9 +513,7 @@ bool ftDataMultiplex::handleRecvData(std::string peerId,
/* called by ftTransferModule */
bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
bool ftDataMultiplex::handleRecvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
{
/**** Find Files *****/
@ -412,8 +533,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
std::cerr << "ftDataMultiplex::handleRecvData() Matched to a Client.";
std::cerr << std::endl;
#endif
locked_handleServerRequest((cit->second).mCreator,
peerId, hash, size, offset, chunksize);
locked_handleServerRequest((cit->second).mCreator, peerId, hash, size, offset, chunksize);
return true;
}
@ -424,8 +544,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
std::cerr << "ftDataMultiplex::handleRecvData() Matched to a Provider.";
std::cerr << std::endl;
#endif
locked_handleServerRequest(sit->second,
peerId, hash, size, offset, chunksize);
locked_handleServerRequest(sit->second, peerId, hash, size, offset, chunksize);
return true;
}
@ -435,9 +554,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
#endif
/* Add to Search Queue */
mSearchQueue.push_back(
ftRequest(FT_DATA_REQ, peerId, hash,
size, offset, chunksize, NULL));
mSearchQueue.push_back( ftRequest(FT_DATA_REQ, peerId, hash, size, offset, chunksize, NULL));
return true;
}
@ -481,6 +598,33 @@ bool ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider,
return false;
}
bool ftDataMultiplex::getClientChunkMap(const std::string& upload_hash,const std::string& peerId,CompressedChunkMap& cmap)
{
bool too_old ;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string,ftFileProvider *>::iterator sit = mServers.find(upload_hash);
if(mServers.end() == sit)
return false ;
sit->second->getClientMap(peerId,cmap,too_old) ;
}
// If the map is too old then we should ask an other map to the peer.
//
if(too_old)
sendChunkMapRequest(peerId,upload_hash);
return true ;
}
bool ftDataMultiplex::sendChunkMapRequest(const std::string& peer_id,const std::string& hash)
{
return mDataSend->sendChunkMapRequest(peer_id,hash);
}
void ftDataMultiplex::deleteServers(const std::list<std::string>& serv)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
@ -491,42 +635,42 @@ void ftDataMultiplex::deleteServers(const std::list<std::string>& serv)
if(mServers.end() != sit)
{
delete sit->second;
// Only delete servers that are not also file creators!
//
if(dynamic_cast<ftFileCreator*>(sit->second) == NULL)
delete sit->second;
mServers.erase(sit);
}
}
}
bool ftDataMultiplex::handleSearchRequest(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
bool ftDataMultiplex::handleSearchRequest(const std::string& peerId, const std::string& hash)
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleSearchRequest(";
std::cerr << peerId << ", " << hash << ", " << size << "...)";
std::cerr << peerId << ", " << hash << "...)";
std::cerr << std::endl;
#endif
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
/* Check for bad requests */
std::map<std::string, time_t>::iterator bit;
if (mUnknownHashs.end() != (bit = mUnknownHashs.find(hash)))
{
#ifdef MPLEX_DEBUG
std::cerr << "ftDataMultiplex::handleSearchRequest(";
std::cerr << " Found Ignore Hash ... done";
std::cerr << std::endl;
#endif
/* We've previously rejected this one, so ignore */
return false;
}
}
// {
// RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
//
// /* Check for bad requests */
// std::map<std::string, time_t>::iterator bit;
// if (mUnknownHashs.end() != (bit = mUnknownHashs.find(hash)))
// {
//
//#ifdef MPLEX_DEBUG
// std::cerr << "ftDataMultiplex::handleSearchRequest(";
// std::cerr << " Found Ignore Hash ... done";
// std::cerr << std::endl;
//#endif
//
// /* We've previously rejected this one, so ignore */
// return false;
// }
// }
/*
@ -535,14 +679,10 @@ bool ftDataMultiplex::handleSearchRequest(std::string peerId,
* (anywhere but remote really)
*/
FileInfo info;
uint32_t hintflags = (RS_FILE_HINTS_CACHE |
RS_FILE_HINTS_EXTRA |
RS_FILE_HINTS_LOCAL |
RS_FILE_HINTS_SPEC_ONLY);
uint32_t hintflags = (RS_FILE_HINTS_CACHE | RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY);
if (mSearch->search(hash, size, hintflags, info))
if (mSearch->search(hash, hintflags, info))
{
#ifdef MPLEX_DEBUG
@ -551,28 +691,35 @@ bool ftDataMultiplex::handleSearchRequest(std::string peerId,
std::cerr << std::endl;
#endif
/* setup a new provider */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
ftFileProvider *provider =
new ftFileProvider(info.path, size, hash);
ftFileProvider *provider = new ftFileProvider(info.path, info.size, hash);
mServers[hash] = provider;
/* handle request finally */
locked_handleServerRequest(provider,
peerId, hash, size, offset, chunksize);
/* now we should should check if any further requests for the same
* file exists ... (can happen with caches!)
*
* but easier to check pre-search....
*/
return true;
}
// Now check wether the required file is actually being downloaded. In such a case,
// setup the file provider to be the file creator itself. Warning: this server should not
// be deleted when not used anymore. We need to restrict this to client peers that are
// not ourself, since the file transfer also handles the local cache traffic (this
// is something to be changed soon!!)
//
if(peerId != mOwnId)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string,ftClient>::const_iterator it(mClients.find(hash)) ;
if(it != mClients.end())
{
mServers[hash] = it->second.mCreator ;
return true;
}
}
return false;
}

32
libretroshare/src/ft/ftdatamultiplex.h
View file

@ -102,23 +102,31 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
/*************** SEND INTERFACE (calls ftDataSend) *******************/
/* Client Send */
bool sendDataRequest(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize);
bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
/* Server Send */
bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
bool sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
/* Server/client Send */
bool sendChunkMapRequest(const std::string& peerId, const std::string& hash) ;
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */
virtual bool recvData(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
virtual bool recvFileMap(const std::string& peerId, const std::string& hash, uint32_t chunk_size, uint32_t nb_chunks, const std::vector<uint32_t>& compressed_map) ;
virtual bool recvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
/// Receive a request for a chunk map
virtual bool recvChunkMapRequest(const std::string& peer_id,const std::string& hash,bool is_client) ;
/// Receive a chunk map
virtual bool recvChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap,bool is_client) ;
/* Server Recv */
virtual bool recvDataRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
virtual bool recvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
// Returns the chunk map from the file uploading client. Also initiates a chunk map request if this
// map is too old. This supposes that the caller will ask again in a few seconds.
//
bool getClientChunkMap(const std::string& upload_hash,const std::string& peer_id,CompressedChunkMap& map) ;
protected:
@ -129,9 +137,11 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
private:
/* Handling Job Queues */
bool handleRecvData(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
bool handleRecvDataRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
bool handleSearchRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
bool handleRecvData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
bool handleRecvDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
bool handleSearchRequest(const std::string& peerId, const std::string& hash);
bool handleRecvClientChunkMapRequest(const std::string& peerId, const std::string& hash) ;
bool handleRecvServerChunkMapRequest(const std::string& peerId, const std::string& hash) ;
/* We end up doing the actual server job here */
bool locked_handleServerRequest(ftFileProvider *provider, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
@ -143,7 +153,7 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
std::list<ftRequest> mRequestQueue;
std::list<ftRequest> mSearchQueue;
std::map<std::string, time_t> mUnknownHashs;
// std::map<std::string, time_t> mUnknownHashs;
ftDataSend *mDataSend;
ftSearch *mSearch;

16
libretroshare/src/ft/ftdbase.cc
View file

@ -37,7 +37,7 @@ ftFiStore::ftFiStore(CacheStrapper *cs, CacheTransfer *cft, NotifyBase *cb_in,
return;
}
bool ftFiStore::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
bool ftFiStore::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{
/* could use hintflags to specify which bits of fileinfo to use additionally.
eg. hintflags & FT_SEARCH_PEER_ID, then only return matching peers + hash.
@ -47,7 +47,7 @@ bool ftFiStore::search(std::string hash, uint64_t size, uint32_t hintflags, File
*/
#ifdef DB_DEBUG
std::cerr << "ftFiStore::search(" << hash << "," << size << "," << hintflags;
std::cerr << "ftFiStore::search(" << hash << "," << hintflags;
std::cerr << ")";
std::cerr << std::endl;
#endif
@ -68,8 +68,8 @@ bool ftFiStore::search(std::string hash, uint64_t size, uint32_t hintflags, File
#endif
bool fullmatch = true;
if (it->size != size)
fullmatch = false;
// if (it->size != size)
// fullmatch = false;
#if 0
@ -125,13 +125,13 @@ ftFiMonitor::ftFiMonitor(CacheStrapper *cs,NotifyBase *cb_in, std::string cached
return;
}
bool ftFiMonitor::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
bool ftFiMonitor::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{
uint64_t fsize;
std::string path;
#ifdef DB_DEBUG
std::cerr << "ftFiMonitor::search(" << hash << "," << size << "," << hintflags;
std::cerr << "ftFiMonitor::search(" << hash << "," << hintflags;
std::cerr << ")";
std::cerr << std::endl;
#endif
@ -269,10 +269,10 @@ ftCacheStrapper::ftCacheStrapper(p3AuthMgr *am, p3ConnectMgr *cm)
}
/* overloaded search function */
bool ftCacheStrapper::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
bool ftCacheStrapper::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{
#ifdef DB_DEBUG
std::cerr << "ftCacheStrapper::search(" << hash << "," << size << "," << hintflags;
std::cerr << "ftCacheStrapper::search(" << hash << "," << hintflags;
std::cerr << ")";
std::cerr << std::endl;
#endif

6
libretroshare/src/ft/ftdbase.h
View file

@ -48,7 +48,7 @@ class ftFiStore: public FileIndexStore, public ftSearch
RsPeerId ownid, std::string cachedir);
/* overloaded search function */
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
};
class ftFiMonitor: public FileIndexMonitor, public ftSearch, public p3Config
@ -57,7 +57,7 @@ class ftFiMonitor: public FileIndexMonitor, public ftSearch, public p3Config
ftFiMonitor(CacheStrapper *cs,NotifyBase *cb_in, std::string cachedir, std::string pid);
/* overloaded search function */
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
/* overloaded set dirs enables config indication */
virtual void setSharedDirectories(std::list<SharedDirInfo> dirList);
@ -81,7 +81,7 @@ class ftCacheStrapper: public CacheStrapper, public ftSearch
ftCacheStrapper(p3AuthMgr *am, p3ConnectMgr *cm);
/* overloaded search function */
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
};

4
libretroshare/src/ft/ftextralist.cc
View file

@ -318,14 +318,14 @@ bool ftExtraList::hashExtraFileDone(std::string path, FileInfo &info)
}
hash = it->second;
}
return search(hash, 0, 0, info);
return search(hash, 0, info);
}
/***
* Search Function - used by File Transfer
*
**/
bool ftExtraList::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
bool ftExtraList::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{
#ifdef DEBUG_ELIST

2
libretroshare/src/ft/ftextralist.h
View file

@ -138,7 +138,7 @@ bool hashExtraFileDone(std::string path, FileInfo &info);
* implementation of ftSearch.
*
**/
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
/***
* Thread Main Loop

84
libretroshare/src/ft/ftfilecreator.cc
View file

@ -38,16 +38,19 @@ ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string hash,
bool ftFileCreator::getFileData(uint64_t offset, uint32_t &chunk_size, void *data)
{
// Only send the data if we actually have it.
//
bool have_it = false ;
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
if (offset + chunk_size > mStart)
{
/* don't have the data */
return false;
}
have_it = chunkMap.isChunkAvailable(offset, chunk_size) ;
}
return ftFileProvider::getFileData(offset, chunk_size, data);
if(have_it)
return ftFileProvider::getFileData(offset, chunk_size, data);
else
return false ;
}
uint64_t ftFileCreator::getRecvd()
@ -225,7 +228,6 @@ int ftFileCreator::locked_notifyReceived(uint64_t offset, uint32_t chunk_size)
/* find the chunk */
std::map<uint64_t, ftChunk>::iterator it = mChunks.find(offset);
// bool isFirst = false;
if (it == mChunks.end())
{
#ifdef FILE_DEBUG
@ -238,11 +240,6 @@ int ftFileCreator::locked_notifyReceived(uint64_t offset, uint32_t chunk_size)
return 0; /* ignoring */
}
// if (it == mChunks.begin())
// {
// isFirst = true;
// }
ftChunk chunk = it->second;
mChunks.erase(it);
@ -256,25 +253,18 @@ int ftFileCreator::locked_notifyReceived(uint64_t offset, uint32_t chunk_size)
else // notify the chunkmap that the slice is finished
chunkMap.dataReceived(chunk.id) ;
// /* update how much has been completed */
// if (isFirst)
// {
// mStart = offset + chunk_size;
// }
// update chunk map
// if (mChunks.size() == 0)
// {
// mStart = mEnd;
// }
/* otherwise there is another earlier block to go
*/
return 1;
}
FileChunksInfo::ChunkStrategy ftFileCreator::getChunkStrategy()
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
return chunkMap.getStrategy() ;
}
void ftFileCreator::setChunkStrategy(FileChunksInfo::ChunkStrategy s)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
@ -289,7 +279,7 @@ void ftFileCreator::setChunkStrategy(FileChunksInfo::ChunkStrategy s)
* But can return size = 0, if we are still waiting for the data.
*/
bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t &offset, uint32_t& size)
bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t &offset, uint32_t& size,bool& source_chunk_map_needed)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
#ifdef FILE_DEBUG
@ -298,17 +288,7 @@ bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hin
std::cerr << std::endl;
locked_printChunkMap();
#endif
/* check start point */
// if(mStart == mSize)
// {
//#ifdef FILE_DEBUG
// std::cerr << "ffc::getMissingChunk() File Done";
// std::cerr << std::endl;
//#endif
// return false;
// }
source_chunk_map_needed = false ;
/* check for freed chunks */
time_t ts = time(NULL);
@ -338,30 +318,17 @@ bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hin
ftChunk chunk ;
if(!chunkMap.getDataChunk(peer_id,size_hint,chunk))
if(!chunkMap.getDataChunk(peer_id,size_hint,chunk,source_chunk_map_needed))
return false ;
// if (mSize - mEnd < chunk)
// chunk = mSize - mEnd;
//
// offset = mEnd;
// mEnd += chunk;
// if (chunk > 0)
// {
#ifdef FILE_DEBUG
std::cerr << "ffc::getMissingChunk() Retrieved new chunk: " << chunk << std::endl ;
// std::cerr << std::endl;
// std::cerr << " mStart: " << mStart << " mEnd: " << mEnd;
// std::cerr << "mSize: " << mSize;
// std::cerr << std::endl;
#endif
mChunks[chunk.offset] = chunk ;
offset = chunk.offset ;
size = chunk.size ;
// }
return true; /* cos more data to get */
}
@ -382,7 +349,6 @@ bool ftFileCreator::locked_printChunkMap()
#endif
/* check start point */
// std::cerr << "Size: " << mSize << " Start: " << mStart << " End: " << mEnd;
std::cerr << "\tOutstanding Chunks:";
std::cerr << std::endl;
@ -394,23 +360,21 @@ bool ftFileCreator::locked_printChunkMap()
return true;
}
void ftFileCreator::loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t strategy)
void ftFileCreator::setAvailabilityMap(const CompressedChunkMap& cmap)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
chunkMap = ChunkMap(mSize,map,chunk_size,chunk_number,FileChunksInfo::ChunkStrategy(strategy)) ;
chunkMap.setAvailabilityMap(cmap) ;
}
void ftFileCreator::storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& strategy)
void ftFileCreator::getAvailabilityMap(CompressedChunkMap& map)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
FileChunksInfo::ChunkStrategy strat ;
chunkMap.buildAvailabilityMap(map,chunk_size,chunk_number,strat) ;
strategy = (uint32_t)strat ;
chunkMap.getAvailabilityMap(map) ;
}
void ftFileCreator::setSourceMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& compressed_map)
void ftFileCreator::setSourceMap(const std::string& peer_id,const CompressedChunkMap& compressed_map)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
@ -421,7 +385,7 @@ void ftFileCreator::setSourceMap(const std::string& peer_id,uint32_t chunk_size,
// - then asking the chunkmap which chunks are being downloaded, but actually shouldn't
// - cancelling them in the ftFileCreator, so that they can be re-asked later to another peer.
//
chunkMap.setPeerAvailabilityMap(peer_id,chunk_size,nb_chunks,compressed_map) ;
chunkMap.setPeerAvailabilityMap(peer_id,compressed_map) ;
}

14
libretroshare/src/ft/ftfilecreator.h
View file

@ -50,16 +50,20 @@ class ftFileCreator: public ftFileProvider
uint64_t getRecvd();
void getChunkMap(FileChunksInfo& info) ;
void setChunkStrategy(FileChunksInfo::ChunkStrategy s) ;
FileChunksInfo::ChunkStrategy getChunkStrategy() ;
/*
* creation functions for FileCreator
*/
// Gets a new variable-sized chunk of size "size_hint" from the given peer id. The returned size, "size" is
// at most equal to size_hint.
// at most equal to size_hint. chunk_map_needed is set if
// - no chunkmap info is available. In such a case, the chunk info is irrelevant and false is returned.
// - the chunk info is too old. In tis case, true is returned, and the chunks info can be used.
//
bool getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size);
bool getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size,bool& is_chunk_map_too_old);
// actually store data in the file, and update chunks info
//
@ -71,12 +75,12 @@ class ftFileCreator: public ftFileProvider
// - getting info about current chunks for the GUI
// - sending availability info to the peers for which we also are a source
//
void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t chunk_strategy) ;
void storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& chunk_strategy) ;
virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
void setAvailabilityMap(const CompressedChunkMap& cmap) ;
// This is called when receiving the availability map from a source peer, for the file being handled.
//
void setSourceMap(const std::string& peer_id,uint32_t chunk_size,uint32_t nb_chunks,const std::vector<uint32_t>& map) ;
void setSourceMap(const std::string& peer_id,const CompressedChunkMap& map) ;
protected:

43
libretroshare/src/ft/ftfileprovider.cc
View file

@ -1,9 +1,12 @@
#include "ftfileprovider.h"
#include "ftchunkmap.h"
#include "util/rsdir.h"
#include <stdlib.h>
#include <stdio.h>
static const time_t UPLOAD_CHUNK_MAPS_TIME = 30 ; // time to ask for a new chunkmap from uploaders in seconds.
ftFileProvider::ftFileProvider(std::string path, uint64_t size, std::string
hash) : mSize(size), hash(hash), file_name(path), fd(NULL),transfer_rate(0),total_size(0)
{
@ -73,6 +76,13 @@ bool ftFileProvider::FileDetails(FileInfo &info)
return true;
}
void ftFileProvider::getAvailabilityMap(CompressedChunkMap& cmap)
{
// We are here because the file we deal with is complete. So we return a plain map.
//
ChunkMap::buildPlainMap(mSize,cmap) ;
}
bool ftFileProvider::getFileData(uint64_t offset, uint32_t &chunk_size, void *data)
{
@ -168,6 +178,39 @@ bool ftFileProvider::getFileData(uint64_t offset, uint32_t &chunk_size, void *da
return 1;
}
void ftFileProvider::setClientMap(const std::string& peer_id,const CompressedChunkMap& cmap)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
std::pair<CompressedChunkMap,time_t>& map_info(clients_chunk_maps[peer_id]) ;
map_info.first = cmap ;
map_info.second = time(NULL) ;
}
void ftFileProvider::getClientMap(const std::string& peer_id,CompressedChunkMap& cmap,bool& map_is_too_old)
{
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
std::map<std::string,std::pair<CompressedChunkMap,time_t> >::iterator it(clients_chunk_maps.find(peer_id)) ;
if(it == clients_chunk_maps.end())
{
clients_chunk_maps[peer_id] = std::pair<CompressedChunkMap,time_t>(CompressedChunkMap(),0) ;
it = clients_chunk_maps.find(peer_id) ;
}
if(time(NULL) - it->second.second > UPLOAD_CHUNK_MAPS_TIME)
{
map_is_too_old = true ;
it->second.second = time(NULL) ; // to avoid re-asking before the TTL
}
else
map_is_too_old = false ;
cmap = it->second.first ;
}
int ftFileProvider::initializeFileAttrs()
{
std::cerr << "ftFileProvider::initializeFileAttrs() Filename: ";

74
libretroshare/src/ft/ftfileprovider.h
View file

@ -37,45 +37,57 @@
class ftFileProvider
{
public:
ftFileProvider(std::string path, uint64_t size, std::string hash);
virtual ~ftFileProvider();
public:
ftFileProvider(std::string path, uint64_t size, std::string hash);
virtual ~ftFileProvider();
virtual bool getFileData(uint64_t offset, uint32_t &chunk_size, void *data);
virtual bool FileDetails(FileInfo &info);
std::string getHash();
uint64_t getFileSize();
bool fileOk();
virtual bool getFileData(uint64_t offset, uint32_t &chunk_size, void *data);
virtual bool FileDetails(FileInfo &info);
std::string getHash();
uint64_t getFileSize();
bool fileOk();
void setPeerId(const std::string& id) ;
void setPeerId(const std::string& id) ;
// Provides a client for the map of chunks actually present in the file. If the provider is also
// a file creator, because the file is actually being downloaded, then the map may be partially complete.
// Otherwize, a plain map is returned.
//
virtual void getAvailabilityMap(CompressedChunkMap& cmap) ;
protected:
virtual int initializeFileAttrs(); /* does for both */
// a ftFileProvider feeds a distant peer. To display what the peers already has, we need to store/read this info.
void getClientMap(const std::string& peer_id,CompressedChunkMap& cmap,bool& map_is_too_old) ;
void setClientMap(const std::string& peer_id,const CompressedChunkMap& cmap) ;
uint64_t mSize;
std::string hash;
std::string file_name;
FILE *fd;
protected:
virtual int initializeFileAttrs(); /* does for both */
/*
* Structure to gather statistics FIXME: lastRequestor - figure out a
* way to get last requestor (peerID)
*/
std::string lastRequestor;
uint64_t req_loc;
uint32_t req_size;
time_t lastTS; // used for checking if it's alive
time_t lastTS_t; // used for estimating transfer rate.
uint64_t mSize;
std::string hash;
std::string file_name;
FILE *fd;
// these two are used for speed estimation
float transfer_rate ;
uint32_t total_size ;
/*
* Structure to gather statistics FIXME: lastRequestor - figure out a
* way to get last requestor (peerID)
*/
std::string lastRequestor;
uint64_t req_loc;
uint32_t req_size;
time_t lastTS; // used for checking if it's alive
time_t lastTS_t; // used for estimating transfer rate.
/*
* Mutex Required for stuff below
*/
RsMutex ftcMutex;
// these two are used for speed estimation
float transfer_rate ;
uint32_t total_size ;
// Info about what the downloading peer already has
std::map<std::string,std::pair<CompressedChunkMap,time_t> > clients_chunk_maps ;
/*
* Mutex Required for stuff below
*/
RsMutex ftcMutex;
};

12
libretroshare/src/ft/ftfilesearch.cc
View file

@ -75,12 +75,12 @@ bool ftFileSearch::addSearchMode(ftSearch *search, uint32_t hintflags)
return false;
}
bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
bool ftFileSearch::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{
uint32_t hints, i;
#ifdef DEBUG_SEARCH
std::cerr << "ftFileSearch::search(" << hash << ", " << size;
std::cerr << "ftFileSearch::search(" << hash ;
std::cerr << ", " << hintflags << ");";
std::cerr << std::endl;
#endif
@ -99,7 +99,7 @@ bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, F
std::cerr << i;
std::cerr << std::endl;
#endif
if (search->search(hash, size, hintflags, info))
if (search->search(hash, hintflags, info))
{
#ifdef DEBUG_SEARCH
std::cerr << "ftFileSearch::search() SLOT: ";
@ -155,7 +155,7 @@ bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, F
std::cerr << "ftFileSearch::search() SLOT: " << i;
std::cerr << std::endl;
#endif
if (search->search(hash, size, hintflags, info))
if (search->search(hash, hintflags, info))
{
#ifdef DEBUG_SEARCH
std::cerr << "ftFileSearch::search() SLOT: ";
@ -180,10 +180,10 @@ bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, F
}
bool ftSearchDummy::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
bool ftSearchDummy::search(std::string hash, uint32_t hintflags, FileInfo &info) const
{
#ifdef DEBUG_SEARCH
std::cerr << "ftSearchDummy::search(" << hash << ", " << size;
std::cerr << "ftSearchDummy::search(" << hash ;
std::cerr << ", " << hintflags << ");";
std::cerr << std::endl;
#endif

2
libretroshare/src/ft/ftfilesearch.h
View file

@ -45,7 +45,7 @@ class ftFileSearch: public ftSearch
ftFileSearch();
bool addSearchMode(ftSearch *search, uint32_t hintflags);
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
private:

4
libretroshare/src/ft/ftsearch.h
View file

@ -43,7 +43,7 @@ class ftSearch
ftSearch() { return; }
virtual ~ftSearch() { return; }
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const = 0;
virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const = 0;
};
@ -54,7 +54,7 @@ class ftSearchDummy: public ftSearch
ftSearchDummy() { return; }
virtual ~ftSearchDummy() { return; }
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
virtual bool search(std::string hash, uint32_t hintflags, FileInfo &info) const;
};
#endif

49
libretroshare/src/ft/ftserver.cc
View file

@ -310,21 +310,9 @@ void ftServer::getDwlDetails(std::list<DwlDetails> & details)
mFtDwlQueue->getDwlDetails(details);
}
bool ftServer::FileChunksDetails(const std::string& hash,FileChunksInfo& info)
bool ftServer::FileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info)
{
return mFtController->getFileChunksDetails(hash,info);
//
// // for know put some dummy info. It's for display sake only.
// info.chunk_size = 1024*1024 ;
// info.file_size = 250*info.chunk_size - 123 ; // last chunk is not complete.
// info.chunks.resize(250,FileChunksInfo::CHUNK_DONE) ;
// int n = rand()%150 + 50 ;
// for(int i=0;i<10;++i)
// info.chunks[n+i] = FileChunksInfo::CHUNK_ACTIVE ;
// for(int i=n+10;i<250;++i)
// info.chunks[i] = FileChunksInfo::CHUNK_OUTSTANDING ;
//
// return true ;
return mFtController->getFileDownloadChunksDetails(hash,info);
}
/* Directory Handling */
@ -360,6 +348,11 @@ bool ftServer::FileDownloads(std::list<std::string> &hashs)
//return mFtDataplex->FileDownloads(hashs);
}
bool ftServer::FileUploadChunksDetails(const std::string& hash,const std::string& peer_id,CompressedChunkMap& cmap)
{
return mFtDataplex->getClientChunkMap(hash,peer_id,cmap);
}
bool ftServer::FileUploads(std::list<std::string> &hashs)
{
return mFtDataplex->FileUploads(hashs);
@ -376,7 +369,7 @@ bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info)
return true ;
if(hintflags & ~(RS_FILE_HINTS_UPLOAD | RS_FILE_HINTS_DOWNLOAD))
if(mFtSearch->search(hash, 0, hintflags, info))
if(mFtSearch->search(hash, hintflags, info))
return true ;
return false;
@ -652,8 +645,7 @@ bool ftServer::loadConfigMap(std::map<std::string, std::string> &configMap)
/***************************************************************/
/* Client Send */
bool ftServer::sendDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize)
bool ftServer::sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendDataRequest(peerId,hash,size,offset,chunksize) ;
@ -680,12 +672,33 @@ bool ftServer::sendDataRequest(std::string peerId, std::string hash,
return true;
}
bool ftServer::sendChunkMapRequest(const std::string& peerId,const std::string& hash)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendChunkMapRequest(peerId,hash) ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
bool ftServer::sendChunkMap(const std::string& peerId,const std::string& hash,const CompressedChunkMap& map)
{
if(mTurtleRouter->isTurtlePeer(peerId))
mTurtleRouter->sendChunkMap(peerId,hash,map) ;
// We only send chunkmap requests to turtle peers. This will be a problem at display time for
// direct friends, so I'll see later whether I code it or not.
return true ;
}
//const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */
//const uint32_t MAX_FT_CHUNK = 16 * 1024; /* 16K */
const uint32_t MAX_FT_CHUNK = 8 * 1024; /* 16K */
/* Server Send */
bool ftServer::sendData(std::string peerId, std::string hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data)
bool ftServer::sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t baseoffset, uint32_t chunksize, void *data)
{
/* create a packet */
/* push to networking part */

14
libretroshare/src/ft/ftserver.h
View file

@ -138,7 +138,9 @@ virtual void getDwlDetails(std::list<DwlDetails> & details);
virtual bool FileDownloads(std::list<std::string> &hashs);
virtual bool FileUploads(std::list<std::string> &hashs);
virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info);
virtual bool FileChunksDetails(const std::string& hash,FileChunksInfo& info) ;
virtual bool FileDownloadChunksDetails(const std::string& hash,FileChunksInfo& info) ;
virtual bool FileUploadChunksDetails(const std::string& hash,const std::string& peer_id,CompressedChunkMap& map) ;
/***
* Extra List Access
@ -196,11 +198,11 @@ virtual bool unshareDownloadDirectory();
/*************** Data Transfer Interface ***********************/
/***************************************************************/
public:
virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
virtual bool sendDataRequest(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize);
virtual bool sendData(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
virtual bool sendDataRequest(const std::string& peerId, const std::string& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
virtual bool sendChunkMapRequest(const std::string& peer_id,const std::string& hash) ;
virtual bool sendChunkMap(const std::string& peer_id,const std::string& hash,const CompressedChunkMap& cmap) ;
/*************** Internal Transfer Fns *************************/
virtual int tick();

8
libretroshare/src/ft/fttransfermodule.cc
View file

@ -327,7 +327,11 @@ bool ftTransferModule::getChunk(const std::string& peer_id,uint32_t size_hint,ui
std::cerr << std::endl;
#endif
bool val = mFileCreator->getMissingChunk(peer_id,size_hint,offset, chunk_size);
bool source_peer_map_needed ;
bool val = mFileCreator->getMissingChunk(peer_id,size_hint,offset, chunk_size,source_peer_map_needed);
if(source_peer_map_needed)
mMultiplexor->sendChunkMapRequest(peer_id, mHash) ;
#ifdef FT_DEBUG
if (val)
@ -338,12 +342,14 @@ bool ftTransferModule::getChunk(const std::string& peer_id,uint32_t size_hint,ui
std::cerr << " size: " << mSize;
std::cerr << " offset: " << offset;
std::cerr << " chunk_size: " << chunk_size;
std::cerr << " peer map needed = " << source_peer_map_needed << std::endl ;
std::cerr << std::endl;
}
else
{
std::cerr << "ftTransferModule::getChunk()";
std::cerr << " Answer: No Chunk Available";
std::cerr << " peer map needed = " << source_peer_map_needed << std::endl ;
std::cerr << std::endl;
}
#endif