Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

added new DL mode: Progressive. This is a random mode that ensures no more than 50MB of free space will be written in the file at once. This should effectively remove the lag when initiating a DL on a big file, while keeping enough entropy to ensure a good dissemination of chunks

Browse Source 
git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@6164 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2013-02-28 20:42:01 +00:00
parent 2d74a3012a
commit 4f1dda2ffb
10 changed files with 76 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
libretroshare/src/ft/ftchunkmap.cc
View File

@ -38,6 +38,7 @@
static const uint32_t SOURCE_CHUNK_MAP_UPDATE_PERIOD = 60 ; //! TTL for chunkmap info static const uint32_t SOURCE_CHUNK_MAP_UPDATE_PERIOD = 60 ; //! TTL for chunkmap info
static const uint32_t INACTIVE_CHUNK_TIME_LAPSE = 300 ; //! TTL for an inactive chunk static const uint32_t INACTIVE_CHUNK_TIME_LAPSE = 300 ; //! TTL for an inactive chunk
static const uint32_t FT_CHUNKMAP_MAX_CHUNK_JUMP = 50 ; //! Maximum chunk jump in progressive DL mode
std::ostream& operator<<(std::ostream& o,const ftChunk& c) std::ostream& operator<<(std::ostream& o,const ftChunk& c)
{ {
@ -74,7 +75,7 @@ ChunkMap::ChunkMap(uint64_t s,bool availability)
++n ; ++n ;
_map.resize(n,FileChunksInfo::CHUNK_OUTSTANDING) ; _map.resize(n,FileChunksInfo::CHUNK_OUTSTANDING) ;
_strategy = FileChunksInfo::CHUNK_STRATEGY_RANDOM ; _strategy = FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE ;
_total_downloaded = 0 ; _total_downloaded = 0 ;
_file_is_complete = false ; _file_is_complete = false ;
#ifdef DEBUG_FTCHUNK #ifdef DEBUG_FTCHUNK
@ -537,27 +538,45 @@ uint32_t ChunkMap::getAvailableChunk(const std::string& peer_id,bool& map_is_too
map_is_too_old = false ;// the map is not too old map_is_too_old = false ;// the map is not too old
uint32_t available_chunks = 0 ; uint32_t available_chunks = 0 ;
uint32_t available_chunks_before_max_dist = 0 ;
uint32_t max_dist = 0; // id of the highest downloaded chunk
for(unsigned int i=0;i<_map.size();++i) for(unsigned int i=0;i<_map.size();++i)
if(_map[i] == FileChunksInfo::CHUNK_OUTSTANDING && (peer_chunks->is_full || peer_chunks->cmap[i])) if(_map[i] == FileChunksInfo::CHUNK_OUTSTANDING)
{ {
if(_strategy == FileChunksInfo::CHUNK_STRATEGY_STREAMING) if(peer_chunks->is_full || peer_chunks->cmap[i])
{ ++available_chunks ;
#ifdef DEBUG_FTCHUNK
std::cerr << "ChunkMap::getAvailableChunk: returning chunk " << i << " for peer " << peer_id << std::endl;
#endif
return i ;
} }
else else
++available_chunks ; {
max_dist = i ;
available_chunks_before_max_dist = available_chunks ;
} }
if(available_chunks > 0) if(available_chunks > 0)
{ {
uint32_t chosen_chunk_number = rand() % available_chunks ; uint32_t chunk_jump ;
uint32_t chosen_chunk_number ;
switch(_strategy)
{
case FileChunksInfo::CHUNK_STRATEGY_STREAMING: chunk_jump = 1 ;
chosen_chunk_number = 0 ;
break ;
case FileChunksInfo::CHUNK_STRATEGY_RANDOM: chunk_jump = _map.size() ;
chosen_chunk_number = rand() % available_chunks ;
break ;
case FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE: chunk_jump = FT_CHUNKMAP_MAX_CHUNK_JUMP ;
chosen_chunk_number = rand() % std::min(available_chunks, available_chunks_before_max_dist+FT_CHUNKMAP_MAX_CHUNK_JUMP) ;
break ;
default:
chunk_jump = _map.size() ;
chosen_chunk_number = 0 ;
}
uint32_t max_chunk = std::min((uint32_t)_map.size(),max_dist + chunk_jump) ;
uint32_t j=0 ; uint32_t j=0 ;
for(uint32_t i=0;i<_map.size();++i) for(uint32_t i=0;i<max_chunk;++i)
if(_map[i] == FileChunksInfo::CHUNK_OUTSTANDING && (peer_chunks->is_full || peer_chunks->cmap[i])) if(_map[i] == FileChunksInfo::CHUNK_OUTSTANDING && (peer_chunks->is_full || peer_chunks->cmap[i]))
{ {
if(j == chosen_chunk_number) if(j == chosen_chunk_number)

7
libretroshare/src/ft/ftcontroller.cc
View File

@ -104,7 +104,7 @@ ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, std::string /
ctrlMutex("ftController"), ctrlMutex("ftController"),
doneMutex("ftController"), doneMutex("ftController"),
mFtActive(false), mFtActive(false),
mDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) mDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE)
{ {
_max_active_downloads = 5 ; // default queue size _max_active_downloads = 5 ; // default queue size
_min_prioritized_transfers = 3 ; _min_prioritized_transfers = 3 ;
@ -2235,6 +2235,11 @@ bool ftController::loadConfigMap(std::map<std::string, std::string> &configMap)
setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) ; setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) ;
std::cerr << "Note: loading default value for chunk strategy: random" << std::endl; std::cerr << "Note: loading default value for chunk strategy: random" << std::endl;
} }
else if(mit->second == "PROGRESSIVE")
{
setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE) ;
std::cerr << "Note: loading default value for chunk strategy: progressive" << std::endl;
}
else else
std::cerr << "**** ERROR ***: Unknown value for default chunk strategy in keymap." << std::endl ; std::cerr << "**** ERROR ***: Unknown value for default chunk strategy in keymap." << std::endl ;
} }

4
libretroshare/src/ft/ftfilecreator.cc
View File

@ -416,10 +416,10 @@ void ftFileCreator::setChunkStrategy(FileChunksInfo::ChunkStrategy s)
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
// Let's check, for safety. // Let's check, for safety.
if(s != FileChunksInfo::CHUNK_STRATEGY_STREAMING && s != FileChunksInfo::CHUNK_STRATEGY_RANDOM) if(s != FileChunksInfo::CHUNK_STRATEGY_STREAMING && s != FileChunksInfo::CHUNK_STRATEGY_RANDOM && s != FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE)
{ {
std::cerr << "ftFileCreator::ERROR: invalid chunk strategy " << s << "!" << " setting default value " << FileChunksInfo::CHUNK_STRATEGY_STREAMING << std::endl ; std::cerr << "ftFileCreator::ERROR: invalid chunk strategy " << s << "!" << " setting default value " << FileChunksInfo::CHUNK_STRATEGY_STREAMING << std::endl ;
s = FileChunksInfo::CHUNK_STRATEGY_STREAMING ; s = FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE ;
} }
#ifdef FILE_DEBUG #ifdef FILE_DEBUG

2
libretroshare/src/retroshare/rstypes.h
View File

@ -311,7 +311,7 @@ class FileChunksInfo
{ {
public: public:
enum ChunkState { CHUNK_CHECKING=3, CHUNK_DONE=2, CHUNK_ACTIVE=1, CHUNK_OUTSTANDING=0 } ; enum ChunkState { CHUNK_CHECKING=3, CHUNK_DONE=2, CHUNK_ACTIVE=1, CHUNK_OUTSTANDING=0 } ;
enum ChunkStrategy { CHUNK_STRATEGY_STREAMING, CHUNK_STRATEGY_RANDOM } ; enum ChunkStrategy { CHUNK_STRATEGY_STREAMING, CHUNK_STRATEGY_RANDOM, CHUNK_STRATEGY_PROGRESSIVE } ;
struct SliceInfo struct SliceInfo
{ {

5
libretroshare/src/tests/ft/ftcrossprovidercreatortest.cc
View File

@ -76,7 +76,7 @@ int main()
std::string fname = "source_tmp.bin" ; std::string fname = "source_tmp.bin" ;
std::string fname_copy_1 = "copy_1_tmp.bin" ; std::string fname_copy_1 = "copy_1_tmp.bin" ;
std::string fname_copy_2 = "copy_2_tmp.bin" ; std::string fname_copy_2 = "copy_2_tmp.bin" ;
uint64_t size = 5000000 ; uint64_t size = 100000000 ;
std::string hash = "" ; std::string hash = "" ;
pthread_t seed = 8;//getpid() ; pthread_t seed = 8;//getpid() ;
@ -92,6 +92,9 @@ int main()
ftFileCreator *client1 = new ftFileCreator(fname_copy_1,size,hash,false) ; ftFileCreator *client1 = new ftFileCreator(fname_copy_1,size,hash,false) ;
ftFileCreator *client2 = new ftFileCreator(fname_copy_2,size,hash,false) ; ftFileCreator *client2 = new ftFileCreator(fname_copy_2,size,hash,false) ;
client1->setChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) ;
client2->setChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE) ;
// 3 - Exchange chunks, and build two copies of the file. // 3 - Exchange chunks, and build two copies of the file.
// //
std::string peer_id_1("client peer id 1") ; std::string peer_id_1("client peer id 1") ;

9
retroshare-gui/src/gui/FileTransferInfoWidget.cpp
View File

@ -260,7 +260,14 @@ void FileTransferInfoWidget::draw(const FileInfo& nfo,const FileChunksInfo& info
y += block_sep ; y += block_sep ;
y += text_height ; painter->drawText(20,y,tr("Number of sources") + ":") ; painter->drawText(tab_size,y,QString::number(info.compressed_peer_availability_maps.size())) ; y += text_height ; painter->drawText(20,y,tr("Number of sources") + ":") ; painter->drawText(tab_size,y,QString::number(info.compressed_peer_availability_maps.size())) ;
y += block_sep ; y += block_sep ;
y += text_height ; painter->drawText(20,y,tr("Chunk strategy") + ":") ; painter->drawText(tab_size,y,(info.strategy==FileChunksInfo::CHUNK_STRATEGY_RANDOM)?"Random":"Streaming") ; y += text_height ; painter->drawText(20,y,tr("Chunk strategy") + ":") ;
switch(info.strategy)
{
case FileChunksInfo::CHUNK_STRATEGY_RANDOM: painter->drawText(tab_size,y,"Random") ; break ;
case FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE: painter->drawText(tab_size,y,"Progressive") ; break ;
default:
case FileChunksInfo::CHUNK_STRATEGY_STREAMING: painter->drawText(tab_size,y,"Streaming") ; break ;
}
y += block_sep ; y += block_sep ;
y += text_height ; painter->drawText(20,y,tr("Transfer type") + ":") ; y += text_height ; painter->drawText(20,y,tr("Transfer type") + ":") ;
if(nfo.transfer_info_flags & RS_FILE_REQ_ANONYMOUS_ROUTING) painter->drawText(tab_size,y,tr("Anonymous F2F")) ; if(nfo.transfer_info_flags & RS_FILE_REQ_ANONYMOUS_ROUTING) painter->drawText(tab_size,y,tr("Anonymous F2F")) ;

7
retroshare-gui/src/gui/TransfersDialog.cpp
View File

@ -397,6 +397,8 @@ TransfersDialog::TransfersDialog(QWidget *parent)
connect(priorityFastAct, SIGNAL(triggered()), this, SLOT(speedFast())); connect(priorityFastAct, SIGNAL(triggered()), this, SLOT(speedFast()));
chunkRandomAct = new QAction(QIcon(IMAGE_PRIORITYAUTO), tr("Random"), this); chunkRandomAct = new QAction(QIcon(IMAGE_PRIORITYAUTO), tr("Random"), this);
connect(chunkRandomAct, SIGNAL(triggered()), this, SLOT(chunkRandom())); connect(chunkRandomAct, SIGNAL(triggered()), this, SLOT(chunkRandom()));
chunkProgressiveAct = new QAction(QIcon(IMAGE_PRIORITYAUTO), tr("Progressive"), this);
connect(chunkProgressiveAct, SIGNAL(triggered()), this, SLOT(chunkProgressive()));
playAct = new QAction(QIcon(IMAGE_PLAY), tr( "Play" ), this ); playAct = new QAction(QIcon(IMAGE_PLAY), tr( "Play" ), this );
connect( playAct , SIGNAL( triggered() ), this, SLOT( openTransfer() ) ); connect( playAct , SIGNAL( triggered() ), this, SLOT( openTransfer() ) );
renameFileAct = new QAction(QIcon(IMAGE_PRIORITYNORMAL), tr("Rename file..."), this); renameFileAct = new QAction(QIcon(IMAGE_PRIORITYNORMAL), tr("Rename file..."), this);
@ -547,6 +549,7 @@ void TransfersDialog::downloadListCustomPopupMenu( QPoint /*point*/ )
QMenu chunkMenu(tr("Chunk strategy"), this); QMenu chunkMenu(tr("Chunk strategy"), this);
chunkMenu.setIcon(QIcon(IMAGE_PRIORITY)); chunkMenu.setIcon(QIcon(IMAGE_PRIORITY));
chunkMenu.addAction(chunkStreamingAct); chunkMenu.addAction(chunkStreamingAct);
chunkMenu.addAction(chunkProgressiveAct);
chunkMenu.addAction(chunkRandomAct); chunkMenu.addAction(chunkRandomAct);
QMenu contextMnu( this ); QMenu contextMnu( this );
@ -1510,6 +1513,10 @@ void TransfersDialog::chunkRandom()
{ {
setChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) ; setChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) ;
} }
void TransfersDialog::chunkProgressive()
{
setChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE) ;
}
void TransfersDialog::setChunkStrategy(FileChunksInfo::ChunkStrategy s) void TransfersDialog::setChunkStrategy(FileChunksInfo::ChunkStrategy s)
{ {
std::set<std::string> items; std::set<std::string> items;

2
retroshare-gui/src/gui/TransfersDialog.h
View File

@ -116,6 +116,7 @@ private slots:
void changeQueuePosition(QueueMove) ; void changeQueuePosition(QueueMove) ;
void chunkRandom(); void chunkRandom();
void chunkProgressive();
void chunkStreaming(); void chunkStreaming();
void showDetailsDialog(); void showDetailsDialog();
@ -166,6 +167,7 @@ private:
QAction *queueTopAct; QAction *queueTopAct;
QAction *queueBottomAct; QAction *queueBottomAct;
QAction *chunkRandomAct; QAction *chunkRandomAct;
QAction *chunkProgressiveAct;
QAction *chunkStreamingAct; QAction *chunkStreamingAct;
QAction *detailsfileAct; QAction *detailsfileAct;
QAction *toggleShowCacheTransfersAct; QAction *toggleShowCacheTransfersAct;

14
retroshare-gui/src/gui/settings/TransferPage.cpp
View File

@ -38,10 +38,12 @@ TransferPage::TransferPage(QWidget * parent, Qt::WFlags flags)
ui._queueSize_SB->setValue(rsFiles->getQueueSize()) ; ui._queueSize_SB->setValue(rsFiles->getQueueSize()) ;
ui._minPrioritized_SB->setValue(rsFiles->getMinPrioritizedTransfers()) ; ui._minPrioritized_SB->setValue(rsFiles->getMinPrioritizedTransfers()) ;
if(rsFiles->defaultChunkStrategy() == FileChunksInfo::CHUNK_STRATEGY_STREAMING) switch(rsFiles->defaultChunkStrategy())
ui._defaultStrategy_CB->setCurrentIndex(0) ; {
else case FileChunksInfo::CHUNK_STRATEGY_STREAMING: ui._defaultStrategy_CB->setCurrentIndex(0) ; break ;
ui._defaultStrategy_CB->setCurrentIndex(1) ; case FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE: ui._defaultStrategy_CB->setCurrentIndex(1) ; break ;
case FileChunksInfo::CHUNK_STRATEGY_RANDOM: ui._defaultStrategy_CB->setCurrentIndex(2) ; break ;
}
ui._diskSpaceLimit_SB->setValue(rsFiles->freeDiskSpaceLimit()) ; ui._diskSpaceLimit_SB->setValue(rsFiles->freeDiskSpaceLimit()) ;
@ -63,9 +65,11 @@ void TransferPage::updateDefaultStrategy(int i)
case 0: rsFiles->setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_STREAMING) ; case 0: rsFiles->setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_STREAMING) ;
break ; break ;
case 1: rsFiles->setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) ; case 2: rsFiles->setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_RANDOM) ;
break ; break ;
case 1: rsFiles->setDefaultChunkStrategy(FileChunksInfo::CHUNK_STRATEGY_PROGRESSIVE) ;
break ;
default: ; default: ;
} }
} }

7
retroshare-gui/src/gui/settings/TransferPage.ui
View File

@ -97,6 +97,11 @@ It is however recommended to leave at least a few slots for cache files.</string
<string>Streaming</string> <string>Streaming</string>
</property> </property>
</item> </item>
<item>
<property name="text">
<string>Progressive</string>
</property>
</item>
<item> <item>
<property name="text"> <property name="text">
<string>Random</string> <string>Random</string>
@ -138,7 +143,7 @@ It is however recommended to leave at least a few slots for cache files.</string
p, li { white-space: pre-wrap; } p, li { white-space: pre-wrap; }
&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Ubuntu'; font-size:11pt; font-weight:400; font-style:normal;&quot;&gt; &lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Ubuntu'; font-size:11pt; font-weight:400; font-style:normal;&quot;&gt;
&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt; font-weight:600;&quot;&gt;RetroShare&lt;/span&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt;&quot;&gt; is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.&lt;/span&gt;&lt;/p&gt; &lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt; font-weight:600;&quot;&gt;RetroShare&lt;/span&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt;&quot;&gt; is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.&lt;/span&gt;&lt;/p&gt;
&lt;p style=&quot;-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-family:'Sans'; font-size:8pt;&quot;&gt;&lt;/p&gt; &lt;p style=&quot;-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-family:'Sans'; font-size:8pt;&quot;&gt;&lt;br /&gt;&lt;/p&gt;
&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt;&quot;&gt;You can separately setup share flags for each shared directory in the shared files dialog to be:&lt;/span&gt;&lt;/p&gt; &lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt;&quot;&gt;You can separately setup share flags for each shared directory in the shared files dialog to be:&lt;/span&gt;&lt;/p&gt;
&lt;ul style=&quot;margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;&quot;&gt;&lt;li style=&quot; font-family:'Sans'; font-size:8pt;&quot; style=&quot; margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Browsable by friends&lt;/span&gt;: files are seen by your friends.&lt;/li&gt; &lt;ul style=&quot;margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;&quot;&gt;&lt;li style=&quot; font-family:'Sans'; font-size:8pt;&quot; style=&quot; margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Browsable by friends&lt;/span&gt;: files are seen by your friends.&lt;/li&gt;
&lt;li style=&quot; font-family:'Sans'; font-size:8pt;&quot; style=&quot; margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Anonymously shared&lt;/span&gt;: files are anonymously reachable through distant F2F tunnels.&lt;/li&gt;&lt;/ul&gt;&lt;/body&gt;&lt;/html&gt;</string> &lt;li style=&quot; font-family:'Sans'; font-size:8pt;&quot; style=&quot; margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Anonymously shared&lt;/span&gt;: files are anonymously reachable through distant F2F tunnels.&lt;/li&gt;&lt;/ul&gt;&lt;/body&gt;&lt;/html&gt;</string>