Implemented load/save of chunk maps for current downloads.

The side effect is that even when stopped, downloads show the correct downloaded size (up to the size of one chunk).



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@1865 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/ft/ftchunkmap.cc

@@ -67,6 +67,35 @@ ChunkMap::ChunkMap(uint64_t s)
 #endif
 }
 
+ChunkMap::ChunkMap(uint64_t file_size,
+							const std::vector<uint32_t>& map,
+							uint32_t chunk_size,
+							uint32_t chunk_number,
+							FileChunksInfo::ChunkStrategy strategy) 
+
+	:_file_size(file_size),_chunk_size(chunk_size),_strategy(strategy)
+{
+#ifdef DEBUG_FTCHUNK
+	std::cerr << "ChunkMap:: loading availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << std::endl ;
+#endif
+
+	_map.clear() ;
+	_map.resize(chunk_number) ;
+	_total_downloaded = 0 ;
+
+	for(uint32_t i=0;i<_map.size();++i)
+	{
+		uint32_t j = i & 31 ;	// i%32
+		uint32_t k = i >> 5 ;	// i/32
+
+		_map[i] = ( (map[k] & (1<<j)) > 0)?(FileChunksInfo::CHUNK_DONE) : (FileChunksInfo::CHUNK_OUTSTANDING) ;
+
+		if(_map[i] == FileChunksInfo::CHUNK_DONE)
+			_total_downloaded += _chunk_size ;
+	}
+}
+
+
 void ChunkMap::dataReceived(const ftChunk::ChunkId& cid)
 {
 	// 1 - find which chunk contains the received data.
@@ -247,5 +276,31 @@ void ChunkMap::getChunksInfo(FileChunksInfo& info) const
 	info.chunks = _map ;
 }
 
+void ChunkMap::buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& strategy) const 
+{
+	chunk_size = _chunk_size ;
+	chunk_number = _map.size() ;
+	strategy = _strategy ;
+
+	map.clear() ;
+	map.reserve((chunk_number >> 5)+1) ;
+
+	uint32_t r=0 ;
+	for(uint32_t i=0;i<_map.size();++i)
+	{
+		uint32_t j = i & 31 ;
+		r |= (_map[i]==FileChunksInfo::CHUNK_DONE)?(1<<j):0 ;
+
+		if(j==31 || i==_map.size()-1)
+		{
+			map.push_back(r);
+			r=0 ;
+		}
+	}
+#ifdef DEBUG_FTCHUNK
+	std::cerr << "ChunkMap:: built availability map of size " << map.size() << ", chunk_size=" << chunk_size << ", chunknumber = " << chunk_number << ", strategy=" << _strategy << std::endl ;
+#endif
+}
+
 
 

libretroshare/src/ft/ftchunkmap.h

@@ -80,6 +80,11 @@ class ChunkMap
 		// Constructor. Decides what will be the size of chunks and how many there will be.
 
 		ChunkMap(uint64_t file_size) ;
+
+		// constructor from saved map info
+		ChunkMap(uint64_t file_size,const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
+
+		// destructor
 		virtual ~ChunkMap() {}
 
       // Returns an slice of data to be asked to the peer within a chunk.
@@ -106,13 +111,14 @@ class ChunkMap
 
 		void setStrategy(FileChunksInfo::ChunkStrategy s) { _strategy = s ; }
 
-#ifdef TO_DO
       // Properly fills an vector of fixed size chunks with availability or download state.
       // chunks is given with the proper number of chunks and we have to adapt to it. This can be used
       // to display square chunks in the gui or display a blue bar of availability by collapsing info from all peers.
 
-      void linearize(FileChunksInfo& info) const ;
+      void buildAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,FileChunksInfo::ChunkStrategy& s) const ;
+		void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,FileChunksInfo::ChunkStrategy s) ;
 
+#ifdef TO_DO
 		// Updates the peer's availablility map
 		//
 		void setPeerAvailabilityMap(const std::string& peer_id,const std::vector<uint32_t>& peer_map) ;
@@ -131,17 +137,17 @@ class ChunkMap
 		uint32_t getAvailableChunk(uint32_t start_location,const std::string& peer_id) const ;
 
 	private:
-		const uint64_t						_file_size ;		// total size of the file in bytes.
+		uint64_t												_file_size ;		// total size of the file in bytes.
-		const uint32_t						_chunk_size ;		// Size of chunks. Common to all chunks.
+		uint32_t												_chunk_size ;		// Size of chunks. Common to all chunks.
-		FileChunksInfo::ChunkStrategy 						_strategy ;			// how do we allocate new chunks
+		FileChunksInfo::ChunkStrategy 				_strategy ;			// how do we allocate new chunks
-		std::map<std::string,Chunk>	_active_chunks_feed ; // vector of chunks being downloaded. Exactly one chunk per peer id.
+		std::map<std::string,Chunk>					_active_chunks_feed ; // vector of chunks being downloaded. Exactly one chunk per peer id.
-		std::map<ChunkNumber,ChunkDownloadInfo> _slices_to_download ; // list of (slice id,slice size) 
+		std::map<ChunkNumber,ChunkDownloadInfo>	_slices_to_download ; // list of (slice id,slice size) 
 
-		std::vector<FileChunksInfo::ChunkState> _map ;				// vector of chunk state over the whole file
+		std::vector<FileChunksInfo::ChunkState>	_map ;				// vector of chunk state over the whole file
 
 		std::map<std::string,std::vector<uint32_t> >	_peers_chunks_availability ;	// what does each source peer have.
 
-		uint64_t								_total_downloaded ;
+		uint64_t												_total_downloaded ;
 };
 
 

libretroshare/src/ft/ftcontroller.cc

@@ -57,6 +57,8 @@
  * #define CONTROL_DEBUG 1
  *****/
 
+static const uint32_t SAVE_TRANSFERS_DELAY = 30	; // save transfer progress every 30 seconds.
+
 ftFileControl::ftFileControl()
 	:mTransfer(NULL), mCreator(NULL),
 	 mState(0), mSize(0), mFlags(0)
@@ -79,7 +81,7 @@ ftFileControl::ftFileControl(std::string fname,
 
 ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, std::string configDir)
 	:CacheTransfer(cs), p3Config(CONFIG_TYPE_FT_CONTROL), mDataplex(dm), mFtActive(false),
-	mTurtle(NULL), mShareDownloadDir(true)
+	mTurtle(NULL), mShareDownloadDir(true),last_save_time(0)
 {
 	/* TODO */
 }
@@ -182,6 +184,13 @@ void ftController::run()
 			doPending = (mFtActive) && (!mFtPendingDone);
 		}
 
+		time_t now = time(NULL) ;
+		if(now - last_save_time > SAVE_TRANSFERS_DELAY)
+		{
+			IndicateConfigChanged() ;
+			last_save_time = now ;
+		}
+
 		if (doPending)
 		{
 			if (!handleAPendingRequest())
@@ -546,17 +555,44 @@ bool 	ftController::isActiveAndNoPending()
 bool	ftController::handleAPendingRequest()
 {
 	ftPendingRequest req;
-  { RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
+	{ 
+		RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
 
-	if (mPendingRequests.size() < 1)
+		if (mPendingRequests.size() < 1)
-	{
+		{
-		return false;
+			return false;
+		}
+		req = mPendingRequests.front();
+		mPendingRequests.pop_front();
 	}
-	req = mPendingRequests.front();
+
-	mPendingRequests.pop_front();
-  }
 	FileRequest(req.mName, req.mHash, req.mSize, req.mDest, req.mFlags, req.mSrcIds);
-	return true;
+
+	{ 
+		// See whether there is a pendign chunk map recorded for this hash.
+		//
+		RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
+
+		std::map<std::string,RsFileTransfer*>::iterator it(mPendingChunkMaps.find(req.mHash)) ;
+
+		if(it != mPendingChunkMaps.end())
+		{
+			RsFileTransfer *rsft = it->second ;
+			std::map<std::string, ftFileControl>::iterator fit = mDownloads.find(rsft->file.hash);
+
+			if((fit==mDownloads.end() || (fit->second).mCreator == NULL))
+			{
+				// This should never happen, because the last call to FileRequest must have created the fileCreator!!
+				//
+				std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
+			}
+			else
+				(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
+
+			delete rsft ;
+			mPendingChunkMaps.erase(it) ;
+		}
+	}
 }
 
 
@@ -1308,8 +1344,7 @@ bool ftController::RequestCacheFile(RsPeerId id, std::string path, std::string h
 	std::list<std::string> ids;
 	ids.push_back(id);
 
-	FileRequest(hash, hash, size, path,
+	FileRequest(hash, hash, size, path, RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids);
-		RS_FILE_HINTS_CACHE | RS_FILE_HINTS_NO_SEARCH, ids);
 
 	return true;
 }
@@ -1422,6 +1457,7 @@ std::list<RsItem *> ftController::saveList(bool &cleanup)
 		//rft->flags = fit->second.mFlags;
 
 		fit->second.mTransfer->getFileSources(rft->allPeerIds.ids);
+		fit->second.mCreator->storeAvailabilityMap(rft->chunk_map,rft->chunk_size,rft->chunk_number,rft->chunk_strategy) ;
 
 		saveData.push_back(rft);
 	}
@@ -1461,17 +1497,27 @@ bool ftController::loadList(std::list<RsItem *> load)
 		}
 		else if (NULL != (rsft = dynamic_cast<RsFileTransfer *>(*it)))
 		{
-//			csoler: I'm suppressing this lock since there is a double lock below
-//					in FileRequest, line 382.
-//  			RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
-//
-
 			/* This will get stored on a waiting list - until the
 			 * config files are fully loaded
 			 */
-			FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize,
+			FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize, rsft->file.path, 0, rsft->allPeerIds.ids);
-				rsft->file.path, 0, rsft->allPeerIds.ids);
 
+			{
+				RsStackMutex mtx(ctrlMutex) ;
+
+				std::map<std::string, ftFileControl>::iterator fit = mDownloads.find(rsft->file.hash);
+
+				if((fit==mDownloads.end() || (fit->second).mCreator == NULL))
+				{
+					std::cerr << "ftController::loadList(): Error: could not find hash " << rsft->file.hash << " in mDownloads list !" << std::endl ;
+					std::cerr << "Storing the map in a wait list." << std::endl ;
+
+					mPendingChunkMaps[rsft->file.hash] = rsft ;
+					continue ;	// i.e. don't delete the item!
+				}
+				else
+					(fit->second).mCreator->loadAvailabilityMap(rsft->chunk_map,rsft->chunk_size,rsft->chunk_number,rsft->chunk_strategy) ;
+			}
 		}
 
 		/* cleanup */

libretroshare/src/ft/ftcontroller.h

@@ -189,6 +189,7 @@ bool    handleAPendingRequest();
 bool    setPeerState(ftTransferModule *tm, std::string id,
                         uint32_t maxrate, bool online);
 
+		time_t last_save_time ;
 	/* pointers to other components */
 
 	ftSearch *mSearch;
@@ -224,6 +225,7 @@ bool    setPeerState(ftTransferModule *tm, std::string id,
 	bool mFtActive;
         bool mFtPendingDone;
 	std::list<ftPendingRequest> mPendingRequests;
+	std::map<std::string,RsFileTransfer*> mPendingChunkMaps ;
 
 	/* share incoming directory */
 	bool mShareDownloadDir;

libretroshare/src/ft/ftfilecreator.cc

@@ -403,4 +403,19 @@ bool ftFileCreator::locked_printChunkMap()
 	return true; 
 }
 
+void ftFileCreator::loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t strategy) 
+{
+	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
+
+	chunkMap = ChunkMap(mSize,map,chunk_size,chunk_number,FileChunksInfo::ChunkStrategy(strategy)) ;
+}
+void ftFileCreator::storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& strategy) 
+{
+	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
+
+	FileChunksInfo::ChunkStrategy strat ;
+	chunkMap.buildAvailabilityMap(map,chunk_size,chunk_number,strat) ;
+	strategy = (uint32_t)strat ;
+}
+
 

libretroshare/src/ft/ftfilecreator.h

@@ -59,6 +59,9 @@ virtual bool 	getFileData(uint64_t offset, uint32_t &chunk_size, void *data);
 	bool	getMissingChunk(const std::string& peer_id,uint32_t size_hint,uint64_t& offset, uint32_t& size);
 	bool 	addFileData(uint64_t offset, uint32_t chunk_size, void *data);
 
+	void loadAvailabilityMap(const std::vector<uint32_t>& map,uint32_t chunk_size,uint32_t chunk_number,uint32_t chunk_strategy) ;
+	void storeAvailabilityMap(std::vector<uint32_t>& map,uint32_t& chunk_size,uint32_t& chunk_number,uint32_t& chunk_strategy) ;
+
 protected:
 
 virtual int initializeFileAttrs(); 

libretroshare/src/serialiser/rsconfigitems.cc

@@ -31,8 +31,6 @@
 #define RSSERIAL_DEBUG 1
 ***/
 
-#define RSSERIAL_DEBUG 1
-
 #include <iostream>
 
 /*************************************************************************/
@@ -198,6 +196,11 @@ uint32_t    RsFileConfigSerialiser::sizeTransfer(RsFileTransfer *item)
 	s += 4; /* trate */
 	s += 4; /* lrate */
 	s += 4; /* ltransfer */
+	s += 4; // chunk_size
+	s += 4; // chunk_number
+	s += 4; // chunk_strategy
+	s += 4; // chunk map size
+	s += 4*item->chunk_map.size(); // chunk_map
 
 	return s;
 }
@@ -240,6 +243,14 @@ bool     RsFileConfigSerialiser::serialiseTransfer(RsFileTransfer *item, void *d
 	ok &= setRawUInt32(data, tlvsize, &offset, item->lrate);
 	ok &= setRawUInt32(data, tlvsize, &offset, item->ltransfer);
 
+	ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_size);
+	ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_number);
+	ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_strategy);
+	ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map.size());
+
+	for(uint32_t i=0;i<item->chunk_map.size();++i)
+		ok &= setRawUInt32(data, tlvsize, &offset, item->chunk_map[i]);
+
 	if (offset != tlvsize)
 	{
 		ok = false;
@@ -299,6 +310,16 @@ RsFileTransfer *RsFileConfigSerialiser::deserialiseTransfer(void *data, uint32_t
 	ok &= getRawUInt32(data, rssize, &offset, &(item->lrate));
 	ok &= getRawUInt32(data, rssize, &offset, &(item->ltransfer));
 
+	ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_size));
+	ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_number));
+	ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_strategy));
+	uint32_t map_size = 0 ;
+	ok &= getRawUInt32(data, rssize, &offset, &map_size);
+
+	item->chunk_map.resize(map_size) ;
+	for(uint32_t i=0;i<map_size;++i)
+		ok &= getRawUInt32(data, rssize, &offset, &(item->chunk_map[i]));
+
 	if (offset != rssize)
 	{
 		/* error */

libretroshare/src/serialiser/rsconfigitems.h

@@ -27,6 +27,7 @@
  */
 
 #include <map>
+#include <vector>
 
 #include "serialiser/rsserial.h"
 #include "serialiser/rstlvbase.h"
@@ -179,31 +180,34 @@ virtual	RsItem *    deserialise(void *data, uint32_t *size);
 class RsFileTransfer: public RsItem
 {
 	public:
-	RsFileTransfer() 
+		RsFileTransfer() :RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG, RS_PKT_TYPE_FILE_CONFIG, RS_PKT_SUBTYPE_FILE_TRANSFER)
-	:RsItem(RS_PKT_VERSION1, RS_PKT_CLASS_CONFIG, 
+		{ 
-		RS_PKT_TYPE_FILE_CONFIG,
+			return; 
-		RS_PKT_SUBTYPE_FILE_TRANSFER)
+		}
-	{ return; }
+		virtual ~RsFileTransfer();
-virtual ~RsFileTransfer();
+		virtual void clear();
-virtual void clear();
+		std::ostream &print(std::ostream &out, uint16_t indent = 0);
-std::ostream &print(std::ostream &out, uint16_t indent = 0);
 
-	RsTlvFileItem file;
+		RsTlvFileItem file;
-	RsTlvPeerIdSet allPeerIds;
+		RsTlvPeerIdSet allPeerIds;
 
-	std::string cPeerId;
+		std::string cPeerId;
 
-	uint16_t state;
+		uint16_t state;
-	uint16_t in;
+		uint16_t in;
 
-	uint64_t transferred;
+		uint64_t transferred;
-	uint32_t crate;
+		uint32_t crate;
-	uint32_t trate;
+		uint32_t trate;
 
+		uint32_t lrate;
+		uint32_t ltransfer;
 
-	uint32_t lrate;
+		// chunk information
-	uint32_t ltransfer;
+		uint32_t chunk_size ;					// common size of chunks
-
+		uint32_t chunk_number ;					// total number of chunks (this is not redondant, cause chunks are compressed)
+		uint32_t chunk_strategy ;				// strategy flags for chunks
+		std::vector<uint32_t> chunk_map ;	// chunk availability (bitwise)
 };
 
 /**************************************************************************/