corrected a bug in chunkmaps that affected file creators being providers at the same time, which could corrupt data. Added some debug info to chunk maps.

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@3673 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/ft/ftchunkmap.cc

@@ -1,3 +1,32 @@
+/*
+ * libretroshare/src/ft: ftdata.cc
+ *
+ * File Transfer for RetroShare.
+ *
+ * Copyright 2010 by Cyril Soler
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "csoler@users.sourceforge.net".
+ *
+ */
+
+/********
+ * #define DEBUG_FTCHUNK 1
+ *********/
+
 #ifdef DEBUG_FTCHUNK
 #include <assert.h>
 #endif
@@ -54,6 +83,10 @@ ChunkMap::ChunkMap(uint64_t s)
 	std::cerr << "   Strategy: " << _strategy << std::endl ;
 	std::cerr << "   ChunkSize: " << _chunk_size << std::endl ;
 	std::cerr << "   Number of Chunks: " << n << std::endl ;
+	std::cerr << "   Data: " ;
+	for(int i=0;i<_map.size();++i)
+		std::cerr << _map[i] ;
+	std::cerr << std::endl ;
 #endif
 }
 
@@ -252,16 +285,24 @@ bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const
 	uint32_t chunk_number_start = offset/(uint64_t)_chunk_size ;
 	uint32_t chunk_number_end = (offset+(uint64_t)chunk_size)/(uint64_t)_chunk_size ;
 
-	if((offset+(uint64_t)chunk_size) % (uint64_t)_chunk_size == 0)
-		--chunk_number_end ;
+	if((offset+(uint64_t)chunk_size) % (uint64_t)_chunk_size != 0)
+		++chunk_number_end ;
 
 	// It's possible that chunk_number_start==chunk_number_end+1, but for this we need to have
 	// chunk_size=0, and offset%_chunk_size=0, so the response "true" is still valid.
 	//
-	for(uint32_t i=chunk_number_start;i!=chunk_number_end;++i)
+	for(uint32_t i=chunk_number_start;i<chunk_number_end;++i)
 		if(_map[i] != FileChunksInfo::CHUNK_DONE)
+		{
+#ifdef DEBUG_FTCHUNK
+			std::cerr << "ChunkMap::isChunkAvailable(): (" << offset << "," << chunk_size << ") is not available" << std::endl; 
+#endif
 			return false ;
+		}
 
+#ifdef DEBUG_FTCHUNK
+	std::cerr << "ChunkMap::isChunkAvailable(): (" << offset << "," << chunk_size << ") is available" << std::endl; 
+#endif
 	return true ;
 }
 

libretroshare/src/ft/ftcontroller.cc

@@ -63,7 +63,6 @@
  * #define CONTROL_DEBUG 1
  * #define DEBUG_DWLQUEUE 1
  *****/
-//#define CONTROL_DEBUG 1
 
 static const uint32_t SAVE_TRANSFERS_DELAY 			= 61	; // save transfer progress every 61 seconds.
 static const uint32_t INACTIVE_CHUNKS_CHECK_DELAY 	= 60	; // time after which an inactive chunk is released
@@ -1946,7 +1945,12 @@ std::list<RsItem *> ftController::saveList(bool &cleanup)
 
 		/* ignore callback ones */
 		if (fit->second->mDoCallback)
+		{
+#ifdef CONTROL_DEBUG
+			std::cerr << "ftcontroller::saveList(): Not saving (callback) file entry " << fit->second->mName << ", " << fit->second->mHash << ", " << fit->second->mSize << std::endl ;
+#endif
 			continue;
+		}
 
 		if ((fit->second)->mCreator->finished())
 			continue;
@@ -2031,6 +2035,9 @@ bool ftController::loadList(std::list<RsItem *> load)
 			/* This will get stored on a waiting list - until the
 			 * config files are fully loaded
 			 */
+#ifdef CONTROL_DEBUG
+			std::cerr << "ftController::loadList(): requesting " << rsft->file.name << ", " << rsft->file.hash << ", " << rsft->file.filesize << std::endl ;
+#endif
 			FileRequest(rsft->file.name, rsft->file.hash, rsft->file.filesize, rsft->file.path, rsft->flags, rsft->allPeerIds.ids);
 
 			{

libretroshare/src/ft/ftfilecreator.cc

@@ -42,12 +42,21 @@ bool ftFileCreator::getFileData(const std::string& peer_id,uint64_t offset, uint
 {
 	// Only send the data if we actually have it.
 	//
+#ifdef FILE_DEBUG
+	std::cerr << "ftFileCreator::getFileData(). Asked for offset=" << offset << ", size=" << chunk_size << std::endl ;
+#endif
 	bool have_it = false ;
 	{
 		RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
 
 		have_it = chunkMap.isChunkAvailable(offset, chunk_size) ;
 	}
+#ifdef FILE_DEBUG
+	if(have_it)
+		std::cerr << "ftFileCreator::getFileData(). Have it" << std::endl ;
+	else
+		std::cerr << "ftFileCreator::getFileData(). Don't have it" << std::endl ;
+#endif
 
 	if(have_it)
 		return ftFileProvider::getFileData(peer_id,offset, chunk_size, data);
@@ -433,6 +442,13 @@ void ftFileCreator::setAvailabilityMap(const CompressedChunkMap& cmap)
 	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
 
 	chunkMap.setAvailabilityMap(cmap) ;
+#ifdef FILE_DEBUG
+	std::cerr << "ftFileCreator: setting chunkmap for hash " << hash << ": " ;
+
+	for(uint32_t i=0;i<cmap._map.size();++i)
+		std::cerr << (void*)cmap._map[i] ;
+	std::cerr << std::endl ;
+#endif
 }
 
 void ftFileCreator::getAvailabilityMap(CompressedChunkMap& map) 

libretroshare/src/ft/ftfileprovider.cc

@@ -9,6 +9,9 @@
 #include "util/rswin.h"
 #endif // WINDOWS_SYS
 
+/********
+* #define DEBUG_FT_FILE_PROVIDER 1
+********/
 
 static const time_t UPLOAD_CHUNK_MAPS_TIME = 20 ;	// time to ask for a new chunkmap from uploaders in seconds.
 

libretroshare/src/retroshare/rstypes.h

@@ -284,7 +284,7 @@ class CompressedChunkMap ;
 class FileChunksInfo
 {
 	public:
-		enum ChunkState { CHUNK_DONE, CHUNK_ACTIVE, CHUNK_OUTSTANDING } ;
+		enum ChunkState { CHUNK_DONE=2, CHUNK_ACTIVE=1, CHUNK_OUTSTANDING=0 } ;
 		enum ChunkStrategy { CHUNK_STRATEGY_STREAMING, CHUNK_STRATEGY_RANDOM } ;
 
 		uint64_t file_size ;					// real size of the file

libretroshare/src/tests/ft/ftcrossprovidercreatortest.cc

@@ -15,18 +15,25 @@ static void transfer(ftFileProvider *server, const std::string& server_peer_id,f
 
 int main()
 {
-	// We create two file creators and feed them with a file both from elsewhere and from each others, as file providers.
-	// This test should check that no data race occurs while reading/writting data and exchanging chunks.
-	//
+	// We create two file creators and feed them with a file both from elsewhere
+	// and from each others, as file providers.  This test should check that no
+	// data race occurs while reading/writting data and exchanging chunks.  The
+	// test is designed so that servers do not always have the required chunk,
+	// meaning that many requests can't be fullfilled. This is a good way to
+	// test if chunks availability is correctly handled.
 
 	// 1 - Create a temporary file, compute its hash
 	//
 	std::string fname = "source_tmp.bin" ;
 	std::string fname_copy_1 = "copy_1_tmp.bin" ;
 	std::string fname_copy_2 = "copy_2_tmp.bin" ;
-	uint64_t size = 55766278 ;
+	uint64_t size = 5560000 ;
 	std::string hash = "" ;
 
+	pthread_t seed = 6;//getpid() ;
+
+	std::cerr << "Seeding random generator with " << seed << std::endl ;
+	srand48(seed) ;
 	createTmpFile(fname,size,hash) ;
 
 	// 2 - Create one file provider for this file, and 2 file creators
@@ -51,36 +58,42 @@ int main()
 		std::string tmpserver_pid ;
 		std::string tmpclient_pid ;
 		
-		// choose client and server, randomly
+		// choose client and server, randomly, provided that they are not finished.
 		//
-		if(lrand48()&1)
+		if((!client1->finished()) && (client2->finished() || (lrand48()&1)))
 		{
+			//std::cerr << "###### Client 1 asking to " ;
 			tmpclient = client1 ;
 			tmpclient_pid = peer_id_1 ;
 
 			if(lrand48()&1)
 			{
+				//std::cerr << "Client 2 " ;
 				tmpserver = client2 ;
 				tmpserver_pid = peer_id_2 ;
 			}
 			else
 			{
+				//std::cerr << "Server ";
 				tmpserver = server ;
 				tmpserver_pid = server_id ;
 			}
 		}
-		else
+		else // client 2 is necessarily unfinished there.
 		{
+			//std::cerr << "###### Client 2 asking to " ;
 			tmpclient = client2 ;
 			tmpclient_pid = peer_id_2 ;
 			
 			if(lrand48()&1)
 			{
+				//std::cerr << "Client 1 " ;
 				tmpserver = client1 ;
 				tmpserver_pid = peer_id_1 ;
 			}
 			else
 			{
+				//std::cerr << "Server " ;
 				tmpserver = server ;
 				tmpserver_pid = server_id ;
 			}
@@ -90,6 +103,9 @@ int main()
 		// 2 - transfer from the server to the client.
 
 		transfer(tmpserver,tmpserver_pid,tmpclient,tmpclient_pid) ;
+
+		printf("Client1: %08lld, Client2: %08lld, transfer from %s to %s\r",client1->getRecvd(),client2->getRecvd(),tmpserver_pid.c_str(),tmpclient_pid.c_str()) ;
+		fflush(stdout) ;
 	}
 
 	// hash the received data
@@ -117,14 +133,33 @@ void transfer(ftFileProvider *server, const std::string& server_peer_id,ftFileCr
 	uint32_t chunk_size = 0;
 	bool toOld = false ;
 
+	//std::cerr << " for a pending chunk of size " << size_hint ;
+
 	if(! client->getMissingChunk(server_peer_id, size_hint, offset, chunk_size, toOld))
 		return ;
 
+	//std::cerr << "###### Got chunk " << offset << ", size=" << chunk_size << std::endl ;
+
 	void *data = malloc(chunk_size) ;
+	//std::cerr << "###### Asking data " << offset << ", size=" << chunk_size << std::endl ;
 
 	if( server->getFileData(client_peer_id,offset,chunk_size,data) )
+	{
 		client->addFileData(offset,chunk_size,data) ;
+		//std::cerr << "###### Added data " << offset << ", size=" << chunk_size << std::endl ;
+	}
 
+	//std::cerr << ", completion=" << client->getRecvd() << std::endl ;
+
+	// Normally this can't be true, because the ChunkMap class assumes availability for non turtle 
+	// peers.
+	if(toOld)
+	{
+		//std::cerr << "###### Map is too old. Transferring" << std::endl ;
+		CompressedChunkMap map ;
+		server->getAvailabilityMap(map) ;
+		client->setSourceMap(server_peer_id,map) ;
+	}
 	free(data) ;
 }