Improvements to file transfer:

- corrected 2 potential flaws of swarming causing request of unavailable data (This should normally be taken care of on the server side!)
- improved swarming test code with fault simulation
- only ask CRC32 maps to peers that have a complete file.
 


git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@4947 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/ft/ftchunkmap.cc

@@ -190,12 +190,12 @@ bool ChunkMap::getDataChunk(const std::string& peer_id,uint32_t size_hint,ftChun
 
 	if(it == _active_chunks_feed.end())		
 	{
+		SourceChunksInfo *sci = getSourceChunksInfo(peer_id) ;
+
 		// 0 - Look into other pending chunks and slice from here.
 		//
 		for(std::map<std::string,Chunk>::iterator pit(_active_chunks_feed.begin());pit!=_active_chunks_feed.end();++pit)
 		{
-			SourceChunksInfo *sci = getSourceChunksInfo(pit->first) ;
-
 			if(sci->is_full || sci->cmap[pit->second._start / _chunk_size])
 			{
 				it = pit ;

libretroshare/src/ft/ftchunkmap.h

@@ -84,6 +84,16 @@ class SourceChunksInfo
 		CompressedChunkMap cmap ;	//! map of what the peer has/doens't have
 		time_t TS ;						//! last update time for this info
 		bool is_full ;					//! is the map full ? In such a case, re-asking for it is unnecessary.
+
+		// Returns true if the offset is starting in a mapped chunk.
+		//
+		bool hasData(uint64_t offset,uint32_t fixed_chunk_size) const
+		{
+			if(is_full)
+				return true ;
+
+			return cmap[offset / (uint64_t)fixed_chunk_size ] ;
+		}
 };
 
 class ChunkMap

libretroshare/src/ft/ftfilecreator.cc

@@ -447,17 +447,17 @@ bool ftFileCreator::getMissingChunk(const std::string& peer_id,uint32_t size_hin
 	// 0 - is there a faulting chunk that would need to be asked again ?
 	
 	for(std::map<uint64_t,ftChunk>::iterator it(mChunks.begin());it!=mChunks.end();++it)
-	if(it->second.ts + CHUNK_MAX_AGE < now)
+		if(it->second.ts + CHUNK_MAX_AGE < now && chunkMap.getSourceChunksInfo(peer_id)->hasData(it->second.offset,ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE))
-	{
+		{
-		offset = it->second.offset ;
+			offset = it->second.offset ;
-		size   = it->second.size ;
+			size   = it->second.size ;
-		it->second.ts = now ;
+			it->second.ts = now ;
 
 #ifdef FILE_DEBUG
-		std::cerr << "ftFileCreator::getMissingChunk(): re-askign for chunk that wasn't received: " << offset << " + " << size << std::endl;
+			std::cerr << "ftFileCreator::getMissingChunk(): re-asking for chunk that wasn't received: " << offset << " + " << size << std::endl;
 #endif
-		return true ;
+			return true ;
-	}
+		}
 
 	// 1 - is there an ongoing 1MB chunk for which we need to take a new slice?
 	//
@@ -562,6 +562,13 @@ void ftFileCreator::getAvailabilityMap(CompressedChunkMap& map)
 	chunkMap.getAvailabilityMap(map) ;
 }
 
+bool ftFileCreator::sourceIsComplete(const std::string& peer_id)
+{
+	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
+
+	return chunkMap.getSourceChunksInfo(peer_id)->is_full ;
+}
+
 void ftFileCreator::setSourceMap(const std::string& peer_id,const CompressedChunkMap& compressed_map)
 {
 	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/

libretroshare/src/ft/ftfilecreator.h

@@ -117,6 +117,10 @@ class ftFileCreator: public ftFileProvider
 		//
 		void setSourceMap(const std::string& peer_id,const CompressedChunkMap& map) ;
 
+		// Returns true id the given file source is complete.
+		//
+		bool sourceIsComplete(const std::string& peer_id) ;
+
 	protected:
 
 		virtual int locked_initializeFileAttrs(); 

libretroshare/src/ft/fttransfermodule.cc

@@ -598,6 +598,8 @@ bool ftTransferModule::checkFile()
 			return true ;
 		}
 	}
+
+
 	forceCheck() ;
 	return false ;
 }
@@ -699,18 +701,21 @@ bool ftTransferModule::checkCRC()
 					break ;
 				}
 
-				int n = rand()%(mFileSources.size()) ;
+				bool found = false ;
-				int p=0 ;
-				std::map<std::string,peerInfo>::const_iterator mit ;
-				for(mit = mFileSources.begin();mit != mFileSources.end() && p<n;++mit,++p) ;
 
-#ifdef FT_DEBUG
+				for(std::map<std::string,peerInfo>::const_iterator mit = mFileSources.begin();mit != mFileSources.end();++mit) 
-				std::cerr << "ftTransferModule::checkCRC(): sending CRC map request to source " << mit->first << std::endl ;
+					if(mFileCreator->sourceIsComplete(mit->first))
-#endif
+					{
-				_crcmap_last_asked_time = now ;
+//#ifdef FT_DEBUG
-				_crcreq_source = mit->first ;
+						std::cerr << "ftTransferModule::checkCRC(): sending CRC map request to source " << mit->first << std::endl ;
+//#endif
+						_crcmap_last_asked_time = now ;
+						_crcreq_source = mit->first ;
 
-				mMultiplexor->sendCRC32MapRequest(mit->first,mHash);
+						mMultiplexor->sendCRC32MapRequest(mit->first,mHash);
+						found = true ;
+						break ;
+					}
 			}
 			break ;
 
@@ -750,7 +755,7 @@ bool ftTransferModule::checkCRC()
 				{
 					// We do as if the file is not complete. This way, it finishes properly.
 					//
-					mFlag = FT_TM_FLAG_DOWNLOADING ;
+					mFlag = FT_TM_FLAG_COMPLETE ;	// Transfer is complete.
 					mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING;
 #ifdef FT_DEBUG
 					std::cerr << "ftTransferModule::checkCRC(): Done. CRC check is ok, file is complete." << std::endl ;

libretroshare/src/tests/ft/ftcrossprovidercreatortest.cc

@@ -3,6 +3,7 @@
 
 #include "util/utest.h"
 #include "util/rsdir.h"
+#include "util/rsdiscspace.h"
 #include <stdlib.h>
 
 #include "util/rswin.h"
@@ -13,6 +14,8 @@ INITTEST();
 static void createTmpFile(const std::string& name,uint64_t size,std::string& hash) ;
 static void transfer(ftFileProvider *server, const std::string& server_peer_id,ftFileCreator *client,const std::string& client_peer_id) ;
 
+static const float TRANSFER_FAIL_PROBABILITY = 0.0f ;
+
 int main()
 {
 	// We create two file creators and feed them with a file both from elsewhere
@@ -22,15 +25,17 @@ int main()
 	// meaning that many requests can't be fullfilled. This is a good way to
 	// test if chunks availability is correctly handled.
 
+	RsDiscSpace::setPartialsPath(".") ;
+
 	// 1 - Create a temporary file, compute its hash
 	//
 	std::string fname = "source_tmp.bin" ;
 	std::string fname_copy_1 = "copy_1_tmp.bin" ;
 	std::string fname_copy_2 = "copy_2_tmp.bin" ;
-	uint64_t size = 5560000 ;
+	uint64_t size = 15560000 ;
 	std::string hash = "" ;
 
-	pthread_t seed = 6;//getpid() ;
+	pthread_t seed = 8;//getpid() ;
 
 	std::cerr << "Seeding random generator with " << seed << std::endl ;
 	srand48(seed) ;
@@ -40,8 +45,8 @@ int main()
 	//
 
 	ftFileProvider *server = new ftFileProvider(fname,size,hash) ;
-	ftFileCreator *client1 = new ftFileCreator(fname_copy_1,size,hash) ;
+	ftFileCreator *client1 = new ftFileCreator(fname_copy_1,size,hash,false) ;
-	ftFileCreator *client2 = new ftFileCreator(fname_copy_2,size,hash) ;
+	ftFileCreator *client2 = new ftFileCreator(fname_copy_2,size,hash,false) ;
 
 	// 3 - Exchange chunks, and build two copies of the file.
 	//
@@ -128,7 +133,7 @@ int main()
 
 void transfer(ftFileProvider *server, const std::string& server_peer_id,ftFileCreator *client,const std::string& client_peer_id)
 {
-	uint32_t size_hint = 128 + (lrand48()%8000) ;
+	uint32_t size_hint = 128 + (lrand48()%1000) ;
 	uint64_t offset = 0;
 	uint32_t chunk_size = 0;
 	bool toOld = false ;
@@ -136,30 +141,41 @@ void transfer(ftFileProvider *server, const std::string& server_peer_id,ftFileCr
 	//std::cerr << " for a pending chunk of size " << size_hint ;
 
 	if(! client->getMissingChunk(server_peer_id, size_hint, offset, chunk_size, toOld))
+	{
+		std::cerr << "###### Server does not have any chunk to offer: server is " << server_peer_id << ", client is " << client_peer_id << std::endl ;
+		// Normally this can't be true, because the ChunkMap class assumes availability for non turtle 
+		// peers.
+		if(toOld)
+		{
+			std::cerr << "###### Map is too old. Transferring chunk map" << std::endl ;
+			CompressedChunkMap map ;
+			server->getAvailabilityMap(map) ;
+			client->setSourceMap(server_peer_id,map) ;
+		}	
 		return ;
+	}
 
-	//std::cerr << "###### Got chunk " << offset << ", size=" << chunk_size << std::endl ;
+	std::cerr << "###### Got chunk " << offset << ", size=" << chunk_size << std::endl ;
 
 	void *data = malloc(chunk_size) ;
-	//std::cerr << "###### Asking data " << offset << ", size=" << chunk_size << std::endl ;
+	std::cerr << "###### Asking data " << offset << ", size=" << chunk_size << std::endl ;
 
 	if( server->getFileData(client_peer_id,offset,chunk_size,data) )
 	{
-		client->addFileData(offset,chunk_size,data) ;
+		if(drand48() < TRANSFER_FAIL_PROBABILITY)
-		//std::cerr << "###### Added data " << offset << ", size=" << chunk_size << std::endl ;
+		{
+			std::cerr << "simulating faulting data chunk " << offset << " + " << chunk_size << "." << std::endl;
+		}
+		else
+		{
+			client->addFileData(offset,chunk_size,data) ;
+			std::cerr << "###### Added data " << offset << ", size=" << chunk_size << std::endl ;
+		}
 	}
 
-	//std::cerr << ", completion=" << client->getRecvd() << std::endl ;
+	std::cerr << ", completion=" << client->getRecvd() << std::endl ;
+
 
-	// Normally this can't be true, because the ChunkMap class assumes availability for non turtle 
-	// peers.
-	if(toOld)
-	{
-		//std::cerr << "###### Map is too old. Transferring" << std::endl ;
-		CompressedChunkMap map ;
-		server->getAvailabilityMap(map) ;
-		client->setSourceMap(server_peer_id,map) ;
-	}
 	free(data) ;
 }
 
@@ -194,6 +210,8 @@ void createTmpFile(const std::string& name,uint64_t S,std::string& hash)
 	}
 	fclose(tmpf) ;
 
-	RsDirUtil::hashFile(name,hash) ;
+	uint64_t size ;
+	std::string path ;
+	RsDirUtil::hashFile(name,path,hash,size) ;
 }