corrected minor bugs, improved testing code

git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-SparseFileStorage@6087 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/ft/ftchunkmap.h

@@ -197,10 +197,10 @@ class ChunkMap
 		/// gets lost when force checking the file.
 		void updateTotalDownloaded() ;
 
-	protected:
 		/// handles what size the last chunk has.
 		uint32_t sizeOfChunk(uint32_t chunk_number) const ;
 
+	protected:
 		/// Returns a chunk available for this peer_id, depending on the chunk strategy.
 		//
 		uint32_t getAvailableChunk(const std::string& peer_id,bool& chunk_map_too_old) ;

libretroshare/src/ft/ftcontroller.cc

@@ -1040,7 +1040,6 @@ bool	ftController::handleAPendingRequest()
 				std::cerr << "Hash " << req.mHash << " is in downloads" << std::endl ;
 				std::cerr << "  setting chunk strategy to " << rsft->chunk_strategy << std::endl;
 #endif
-				//(fit->second)->mCreator->setAvailabilityMap(rsft->compressed_chunk_map) ;
 				(fit->second)->mCreator->setAvailabilityMap(rsft->compressed_chunk_map,rsft->data_chunk_ids) ;
 				(fit->second)->mCreator->setChunkStrategy((FileChunksInfo::ChunkStrategy)(rsft->chunk_strategy)) ;
 			}
@@ -2030,6 +2029,7 @@ bool ftController::saveList(bool &cleanup, std::list<RsItem *>& saveData)
 
 		fit->second->mCreator->getAvailabilityMap(rft->compressed_chunk_map) ;
 		rft->chunk_strategy = fit->second->mCreator->getChunkStrategy() ;
+		rft->data_chunk_ids = fit->second->mCreator->getMappedChunks() ;
 
 		saveData.push_back(rft);
 	}
@@ -2145,7 +2145,6 @@ bool ftController::loadList(std::list<RsItem *>& load)
 				}
 				else
 				{
-					//(fit->second)->mCreator->setAvailabilityMap(rsft->compressed_chunk_map) ;
 					(fit->second)->mCreator->setAvailabilityMap(rsft->compressed_chunk_map,rsft->data_chunk_ids) ;
 					(fit->second)->mCreator->setChunkStrategy((FileChunksInfo::ChunkStrategy)(rsft->chunk_strategy)) ;
 				}

libretroshare/src/ft/ftfilecreator.cc

@@ -507,6 +507,8 @@ void ftFileCreator::getChunkMap(FileChunksInfo& info)
 
 		info.pending_slices[n].push_back(si) ;
 	}
+
+	info.chunks_on_disk = getMappedChunks() ;
 }
 
 bool ftFileCreator::locked_printChunkMap()
@@ -611,7 +613,11 @@ void ftFileCreator::forceCheck()
 {
 	RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
 
-	chunkMap.forceCheck(); 
+	std::cerr << "WARNING: forceCheck() for mapped files is not yet implemented." << std::endl;
+	return ;
+
+	forceCheckPartialFile() ;	// calls ftFileMapper, to ask for re-hashing the partial file
+	chunkMap.forceCheck(); 		// sets all chunk to checking mode.
 }
 
 void ftFileCreator::getSourcesList(uint32_t chunk_num,std::vector<std::string>& sources)
@@ -635,11 +641,10 @@ bool ftFileCreator::verifyChunk(uint32_t chunk_number,const Sha1CheckSum& sum)
 	if(!locked_initializeFileAttrs() )
 		return false ;
 
-	static const uint32_t chunk_size = ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE ;
-	unsigned char *buff = new unsigned char[chunk_size] ;
-	uint32_t len ;
+	uint32_t len = chunkMap.sizeOfChunk(chunk_number);
+	unsigned char *buff = new unsigned char[len] ;
 
-	if(fseeko64(fd,(uint64_t)chunk_number * (uint64_t)chunk_size,SEEK_SET)==0 && (len = fread(buff,1,chunk_size,fd)) > 0)
+	if(retrieveData(buff,len,(uint64_t)chunk_number * (uint64_t)ChunkMap::CHUNKMAP_FIXED_CHUNK_SIZE,fd)) // if(fseeko64(fd,(uint64_t)chunk_number * (uint64_t)chunk_size,SEEK_SET)==0 && (len = fread(buff,1,chunk_size,fd)) > 0)
 	{
 		Sha1CheckSum comp = RsDirUtil::sha1sum(buff,len) ;
 

libretroshare/src/ft/ftfilemapper.cc

@@ -269,7 +269,7 @@ void ftFileMapper::initMappedChunks(uint64_t file_size,const CompressedChunkMap&
 
 	// 0 - retro-compatibility. First check that the number of chunks in both maps co-incide.
 
-	if(data_chunk_ids.size() != count)
+	if(data_chunk_ids.size() < count)
 	{
 		std::cerr << "(II) ftFileMapper::initMappedChunks(): file has " << count << " chunks on disk, but no mapping. Assuming it's an unfragmented file (backward compatibility)!" << std::endl;
 		_data_chunk_ids.resize(numck) ;

libretroshare/src/ft/ftfilemapper.h

@@ -80,9 +80,14 @@ class ftFileMapper
 		//
 		bool retrieveData(void *data, uint32_t data_size, uint64_t offset,FILE *fd) ;
 
+		// Returns the number of each chunk that is mapped to place 0,1,2,... on the disk.
+		//
+		const std::vector<uint32_t>& getMappedChunks() const { return _data_chunk_ids ; }
+
 		// debug
 		void print() const ;
 
+		virtual void forceCheckPartialFile() {}
 	private:
 		uint64_t _file_size ;	// size of the file
 		uint32_t _chunk_size ;	// size of chunks

libretroshare/src/retroshare/rstypes.h

@@ -337,6 +337,9 @@ class FileChunksInfo
 		// The list of pending requests, chunk per chunk (by chunk id)
 		//
 		std::map<uint32_t, std::vector<SliceInfo> > pending_slices ;
+
+		// chunk order on the disk
+		std::vector<uint32_t> chunks_on_disk ;
 };
 
 class CompressedChunkMap

libretroshare/src/tests/ft/ftfilemappertest.cc

@@ -5,6 +5,7 @@
 #include <util/utest.h>
 #include <util/rsdir.h>
 #include <common/argstream.h>
+#include <common/testutils.h>
 #include <stdlib.h>
 #include <iostream>
 #include <stdint.h>
@@ -17,6 +18,7 @@ int main(int argc,char *argv[])
 
 	argstream as(argc,argv) ;
 	int random_seed = 0 ;
+	int nb_virtual_peers = 5 ;
 	uint64_t size = 1024*1024*12+234;//4357283      ; // some size. Not an integer number of chunks
 
 	as >> parameter('r',"random-seed",random_seed,"Random seed for the test. Use the same seed to get the same behavior.",false) 
@@ -40,15 +42,10 @@ int main(int argc,char *argv[])
 	void *membuf = malloc(size) ;
 	CHECK(membuf != NULL) ;
 
-	for(int i=0;i<size;++i)
+	for(uint32_t i=0;i<size;++i)
 		((char*)membuf)[i] = lrand48() & 0xff ;
 
-	// also write it to disk
-	
-	FILE *f = fopen(input_file.c_str(),"w") ;
-	CHECK(f != NULL) ;
-	CHECK(fwrite(membuf,size,1,f) == 1) ;
-	fclose(f) ;
+	std::string sha1_1 = RsDirUtil::sha1sum((unsigned char *)membuf,size).toStdString() ;
 		
 	// 1 - allocate a chunkmap for this file
 	//
@@ -65,7 +62,16 @@ int main(int argc,char *argv[])
 	ftChunk chunk ;
 	bool source_map_needed ;
 
-	while(chunk_map.getDataChunk("virtual peer",1024*200+(lrand48()%1024),chunk,source_map_needed))
+	std::vector<std::string> virtual_peers(nb_virtual_peers) ;
+
+	for(uint32_t i=0;i<virtual_peers.size();++i)
+	{
+		std::string peer_id = TestUtils::createRandomSSLId() ;
+		virtual_peers[i] = peer_id ;
+		std::cerr << "Allocating new virtual peer " << peer_id << std::endl;
+	}
+
+	while(chunk_map.getDataChunk(virtual_peers[lrand48()%nb_virtual_peers],1024*200+(lrand48()%1024),chunk,source_map_needed))
 	{
 		//std::cerr << "Got chunk " << chunk.offset << " + " << chunk.size << " from chunkmap." << std::endl;
 
@@ -81,19 +87,19 @@ int main(int argc,char *argv[])
 
 	// Check the sha1 of both source and destination.
 	//
-	std::string sha1_1,sha1_2 ;
-	uint64_t size_1,size_2 ;
+	std::string sha1_2 ;
+	uint64_t size_1 = size ;
+	uint64_t size_2 ;
 
-	RsDirUtil::getFileHash( input_file,sha1_1,size_1) ;
 	RsDirUtil::getFileHash(output_file,sha1_2,size_2) ;
 
-	std::cerr << "Computed hash of file\t " << input_file << "\t :\t" << sha1_1 << ", size=" << size_1 << std::endl;
-	std::cerr << "Computed hash of file\t " <<output_file << "\t :\t" << sha1_2 << ", size=" << size_2 << std::endl;
+	std::cerr << "Computed hash of file\t " <<  input_file << "\t :\t" << sha1_1 << ", size=" << size_1 << std::endl;
+	std::cerr << "Computed hash of file\t " << output_file << "\t :\t" << sha1_2 << ", size=" << size_2 << std::endl;
 
 	CHECK(size_1 == size_2) ;
 	CHECK(sha1_1 == sha1_2) ;
 
-	FINALREPORT("RsTlvItem Stack Tests");
+	FINALREPORT("FtFileMapper Tests");
 
 	return TESTRESULT();
 }

retroshare-gui/src/gui/TransfersDialog.cpp

@@ -1040,7 +1040,7 @@ void TransfersDialog::insertTransfers()
 			uint32_t chunk_size = 1024*1024 ;
 			uint32_t nb_chunks = (uint32_t)((info.size + (uint64_t)chunk_size - 1) / (uint64_t)(chunk_size)) ;
 
-			uint32_t filled_chunks = pinfo.cmap.filledChunks(nb_chunks) ;
+			uint32_t filled_chunks = pinfo.cmap.countFilledChunks(nb_chunks) ;
 			pinfo.type = FileProgressInfo::UPLOAD_LINE ;
 			pinfo.nb_chunks = pinfo.cmap._map.empty()?0:nb_chunks ;