- implemented post-download hash re-check. For now, if the hash does not match, the download is canceled, but in the near future, per-chunk comparison wil occur

.
- corrected a bug that caused file copy error: a closeFile() was missing when the file is complete. Because of delays in fwrite, the file would not be always co
mplete, nor exist at all for small files (e.g. cache files), which in the later case caused the copy error.

Warning: needs a make clean in libretroshare to recompile.



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@3261 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/dbase/fimonitor.cc

@@ -42,7 +42,6 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 #include <unistd.h>
-#include <openssl/sha.h>
 #include <stdio.h>
 
 //***********
@@ -640,7 +639,7 @@ void FileIndexMonitor::hashFiles(const std::vector<DirContentToHash>& to_hash)
 
 			FileEntry fe(to_hash[i].fentries[j]) ;	// copied, because hashFile updates the hash member
 
-			if (hashFile(to_hash[i].realpath, fe))
+			if(RsDirUtil::hashFile(to_hash[i].realpath + "/" + to_hash[i].fentries[j].name, fe.hash))
 			{
 				RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/
 
@@ -977,66 +976,6 @@ std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const
 	return realroot;
 }
 
-
-
-bool FileIndexMonitor::hashFile(std::string fullpath, FileEntry& fent)
-{
-	std::string f_hash = fullpath + "/" + fent.name;
-	FILE *fd;
-	int  len;
-	SHA_CTX *sha_ctx = new SHA_CTX;
-	unsigned char sha_buf[SHA_DIGEST_LENGTH];
-	unsigned char gblBuf[512];
-
-#ifdef FIM_DEBUG
-	std::cerr << "File to hash = " << f_hash << std::endl;
-#endif
-
-#ifdef WINDOWS_SYS
-        std::wstring wf_hash;
-        librs::util::ConvertUtf8ToUtf16(f_hash, wf_hash);
-        if (NULL == (fd = _wfopen(wf_hash.c_str(), L"rb")))
-            return false;
-#else
-        if (NULL == (fd = fopen64(f_hash.c_str(), "rb")))
-            return false;
-#endif
-
-	SHA1_Init(sha_ctx);
-	while((len = fread(gblBuf,1, 512, fd)) > 0)
-	{
-		SHA1_Update(sha_ctx, gblBuf, len);
-	}
-
-	/* reading failed for some reason */
-	if (ferror(fd))
-	{
-#ifdef FIM_DEBUG
-		std::cerr << "read error !!" << std::endl;
-#endif
-		delete sha_ctx;
-		fclose(fd);
-		return false;
-	}
-
-	SHA1_Final(&sha_buf[0], sha_ctx);
-
-	/* TODO: Actually we should store the hash data as binary ...
-	 * but then it shouldn't be put in a string.
-	 */
-
-        std::ostringstream tmpout;
-	for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
-	{
-		tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]);
-	}
-	fent.hash = tmpout.str();
-
-	delete sha_ctx;
-	fclose(fd);
-	return true;
-}
-
 int FileIndexMonitor::RequestDirDetails(std::string uid, std::string path, DirDetails &details) const
 {
 	/* lock it up */

libretroshare/src/ft/ftcontroller.cc

@@ -1652,6 +1652,9 @@ bool 	ftController::FileDetails(std::string hash, FileInfo &info)
 	if(it->second->mState == ftFileControl::PAUSED)
 		info.downloadStatus = FT_STATE_PAUSED ;
 
+	if((!completed) && it->second->mTransfer->isCheckingHash())
+		info.downloadStatus = FT_STATE_CHECKING_HASH ;
+
 	info.tfRate = totalRate;
 	info.size = (it->second)->mSize;
 

libretroshare/src/ft/ftfilecreator.cc

@@ -2,6 +2,7 @@
 #include <errno.h>
 #include <stdio.h>
 #include <util/rsdiscspace.h>
+#include <util/rsdir.h>
 
 /*******
  * #define FILE_DEBUG 1
@@ -102,6 +103,8 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data
 	if(!RsDiscSpace::checkForDiscSpace(RS_PARTIALS_DIRECTORY))
 		return false ;
 
+	bool complete = false ;
+	{
 		RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
 
 		if (fd == NULL)
@@ -146,6 +149,16 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data
 		 */
 		locked_notifyReceived(offset,chunk_size);
 
+		complete = chunkMap.isComplete();
+	}
+	if(complete)
+	{
+#ifdef FILE_DEBUG
+		std::cerr << "ftFileCreator::addFileData() File is complete: closing" << std::endl ;
+#endif
+		closeFile();
+	}
+  
 	/* 
 	 * FIXME HANDLE COMPLETION HERE - Any better way?
 	 */
@@ -451,3 +464,17 @@ bool ftFileCreator::finished()
 	return chunkMap.isComplete() ;
 }
 
+bool ftFileCreator::hashReceivedData(std::string& hash)
+{
+	RsStackMutex stack(ftcMutex) ;
+
+	return RsDirUtil::hashFile(file_name,hash) ;
+}
+
+#include <stdexcept>
+bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref)
+{
+	// still to be implemented.
+	throw std::runtime_error(std::string("Unimplemented function ") + __PRETTY_FUNCTION__) ;
+	return true ;
+}

libretroshare/src/ft/ftfilecreator.h

@@ -57,6 +57,17 @@ class ftFileCreator: public ftFileProvider
 		void setChunkStrategy(FileChunksInfo::ChunkStrategy s) ;
 		FileChunksInfo::ChunkStrategy getChunkStrategy() ;
 
+		// Computes a sha1sum of the partial file, to check that the data is overall consistent.
+		//
+		bool hashReceivedData(std::string& hash) ;
+
+		// Computes a CRC32 map of all chunks, for comparison with reference, and re-starting invalid chunks.
+		//
+		bool CRC32ReceivedData(CRC32Map& crc_map) ;
+
+		// Checks the CRC32 of all chunks against the given CRC32 map. Re-flag the bad chunks as being void.
+		//
+		bool crossCheckChunkMap(const CRC32Map& crc_map) ;
 		/* 
 		 * creation functions for FileCreator 
 		 */

libretroshare/src/ft/fttransfermodule.cc

@@ -58,8 +58,12 @@ const int32_t FT_TM_FAST_RTT    = 1.0;
 const int32_t FT_TM_STD_RTT     = 5.0;
 const int32_t FT_TM_SLOW_RTT    = 9.0;
 
+#define FT_TM_FLAG_DOWNLOADING 0
+#define FT_TM_FLAG_CHECKING    3
+#define FT_TM_FLAG_CHECKING    3
+
 ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm, ftController *c)
-	:mFileCreator(fc), mMultiplexor(dm), mFtController(c), mFlag(0)
+	:mFileCreator(fc), mMultiplexor(dm), mFtController(c), mFlag(FT_TM_FLAG_DOWNLOADING)
 {
   	RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
 
@@ -385,6 +389,9 @@ bool ftTransferModule::queryInactive()
 	if (mFileStatus.stat == ftFileStatus::PQIFILE_INIT)
 		mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING;
 
+	if (mFileStatus.stat == ftFileStatus::PQIFILE_CHECKING)
+		return false ;
+
 	if (mFileStatus.stat != ftFileStatus::PQIFILE_DOWNLOADING)
 	{
 		if (mFileStatus.stat == ftFileStatus::PQIFILE_FAIL_CANCEL)
@@ -398,7 +405,10 @@ bool ftTransferModule::queryInactive()
 		locked_tickPeerTransfer(mit->second);
 	}
 	if(mFileCreator->finished())	// transfer is complete
-		mFlag = 1;      
+	{
+		mFileStatus.stat = ftFileStatus::PQIFILE_CHECKING ;
+		mFlag = 3;      
+	}
 
 	return true; 
 }
@@ -411,6 +421,12 @@ bool ftTransferModule::cancelTransfer()
   return 1;
 }
 
+bool ftTransferModule::cancelFileTransferUpward()
+{
+	if (mFtController)
+		mFtController->FileCancel(mHash);
+	return true;
+}
 bool ftTransferModule::completeFileTransfer()
 {
 #ifdef FT_DEBUG
@@ -463,6 +479,14 @@ int ftTransferModule::tick()
 		  break;
 	  case 2: //file transfer canceled
 		  break;
+	  case 3: 
+		  // Check if file hash matches the hashed data
+		  checkFile() ;
+		  break ;
+	  case 4:
+		  // File is waiting for CRC32 map. Check if received, and re-set matched chunks
+		  checkCRC() ;
+		  break ;
 	  default:
 		  break;
   }
@@ -470,6 +494,67 @@ int ftTransferModule::tick()
   return 0;
 }
 
+bool ftTransferModule::isCheckingHash()
+{
+  	RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
+	return mFlag == 3 ;
+}
+bool ftTransferModule::checkFile()
+{
+	{
+		RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
+#ifdef FT_DEBUG
+		std::cerr << "ftTransferModule::checkFile(): checking File " << mHash << std::endl ;
+#endif
+		std::string hash_check ;
+
+		if(!mFileCreator->hashReceivedData(hash_check))
+		{
+			std::cerr << "ftTransferModule::checkFile(): Impossible to hash received file " << mHash << " for check. What is happenning?" << std::endl ;
+			return false ;
+		}
+
+		if(hash_check == mHash)
+		{
+			mFlag = 1 ;	// Transfer is complete.
+#ifdef FT_DEBUG
+			std::cerr << "ftTransferModule::checkFile(): File verification complete ! Setting mFlag to 1" << std::endl ;
+#endif
+			return true ;
+		}
+	
+		mFlag = 4 ;	// Ask for CRC map. But for now, cancel file transfer.
+		mFileStatus.stat = ftFileStatus::PQIFILE_FAIL_CANCEL ;
+	}
+	cancelFileTransferUpward() ;
+	std::cerr << "(EE) ftTransferModule::checkFile(): File verification failed for hash " << mHash << "! Asking for CRC map. mFlag=4. For now: cancelling file transfer." << std::endl ;
+	//askForCRCMap() ;
+	
+	return false ;
+}
+
+bool ftTransferModule::checkCRC()
+{
+#ifdef FT_DEBUG
+	std::cerr << "ftTransferModule::checkCRC(): looking for CRC32 map." << std::endl ;
+#endif
+
+	if(_crc32map.empty())
+		return false ;
+
+#ifdef FT_DEBUG
+	std::cerr << "ftTransferModule::checkCRC(): got a CRC32 map. Matching chunks..." << std::endl ;
+#endif
+
+	mFileCreator->crossCheckChunkMap(_crc32map) ;
+	mFlag = 0 ;
+
+#ifdef FT_DEBUG
+	std::cerr << "ftTransferModule::checkCRC(): Done. Restarting download." << std::endl ;
+#endif
+	return true ;
+}
+
 
 void ftTransferModule::adjustSpeed()
 {

libretroshare/src/ft/fttransfermodule.h

@@ -106,6 +106,7 @@ public:
 		PQIFILE_NOT_ONLINE,
 		PQIFILE_DOWNLOADING,
 		PQIFILE_COMPLETE,
+		PQIFILE_CHECKING,
 		PQIFILE_FAIL,
 		PQIFILE_FAIL_CANCEL,
 		PQIFILE_FAIL_NOT_AVAIL,
@@ -138,7 +139,9 @@ public:
   bool getPeerState(std::string peerId,uint32_t &state,uint32_t &tfRate);  
   uint32_t getDataRate(std::string peerId);
   bool cancelTransfer();
+  bool cancelFileTransferUpward();
   bool completeFileTransfer();
+  bool isCheckingHash() ;
 
   //interface to multiplex module
   bool recvFileData(std::string peerId, uint64_t offset, 
@@ -164,6 +167,8 @@ private:
   bool locked_recvPeerData(peerInfo &info, uint64_t offset,
 			uint32_t chunk_size, void *data);
   
+  bool checkFile() ;
+  bool checkCRC() ;
   
   /* These have independent Mutexes / are const locally (no Mutex protection)*/
   ftFileCreator *mFileCreator;
@@ -178,10 +183,12 @@ private:
   std::list<std::string>         mOnlinePeers;
   std::map<std::string,peerInfo> mFileSources;
   	
-  uint16_t     mFlag;  //2:file canceled, 1:transfer complete, 0: not complete
+  uint16_t     mFlag;  //2:file canceled, 1:transfer complete, 0: not complete, 3: checking hash, 4: checking chunks
   double desiredRate;
   double actualRate;
 
+  CRC32Map _crc32map ;
+
   ftFileStatus mFileStatus; //used for pause/resume file transfer
 };
 

libretroshare/src/rsiface/rstypes.h

@@ -46,6 +46,7 @@ const uint32_t FT_STATE_DOWNLOADING	= 0x0003 ;
 const uint32_t FT_STATE_COMPLETE 		= 0x0004 ;
 const uint32_t FT_STATE_QUEUED   		= 0x0005 ;
 const uint32_t FT_STATE_PAUSED   		= 0x0006 ;
+const uint32_t FT_STATE_CHECKING_HASH	= 0x0007 ;
 
 // These constants are used by RsDiscSpace
 //
@@ -338,6 +339,23 @@ class CompressedChunkMap
 		std::vector<uint32_t> _map ;
 };
 
+class CRC32Map
+{
+	public:
+		// Build from a file.
+		//
+		CRC32Map(const std::string& fname,uint32_t chunk_size) ;
+		CRC32Map() {}
+
+		// Compares two maps and returns the valid chunks in a compressed chunk map.
+		//
+		friend CompressedChunkMap compare(const CRC32Map& crc1,const CRC32Map& crc2) ;
+
+		bool empty() const { return _map.empty() ; }
+	private:
+		std::vector<uint32_t> _map ;
+};
+
 /* class which encapsulates download details */
 class DwlDetails {
 public:

libretroshare/src/util/rsdir.cc

@@ -36,6 +36,9 @@
 #include <algorithm>
 #include <stdio.h>
 #include <dirent.h>
+#include <openssl/sha.h>
+#include <iomanip>
+#include <sstream>
 
 #include <fstream>
 
@@ -74,6 +77,121 @@ std::string 	RsDirUtil::getTopDir(std::string dir)
 	return top;
 }
 
+class CRC32Table
+{
+	public:
+		inline uint32_t operator[](unsigned char i) const { return _data[i] ; }
+
+		CRC32Table()
+		{
+			_data = new uint32_t[256] ;
+			uint32_t polynmial = 0x04c11db7 ;
+
+			for(uint32_t i=0;i<256;++i)
+			{
+				uint32_t a = reflect(i,8)<<24 ;
+
+				for(uint32_t j=0;j<8;++j)
+					a = (a << 1)^ ( (a&(1<<31))?polynmial:0) ;
+
+				_data[i] = reflect(a,32) ;
+			}
+		}
+		// Swap bits 0-7, 1-6, etc.
+		//
+		uint32_t reflect(uint32_t ref,unsigned char ch)
+		{
+			uint32_t val = 0 ;
+			for(int i=1;i<ch+1;i++,ref>>=1)
+				if(ref & 1)
+					val |= (1 << (ch-i)) ;
+			return val ;
+		}
+		~CRC32Table() { delete[] _data ; }
+
+	private:
+		uint32_t *_data ;
+};
+
+uint32_t rs_CRC32(const unsigned char *data,uint32_t len) 
+{
+	static const CRC32Table crc32_table ;
+	uint32_t a = 0xffffffff ;
+
+	for(const unsigned char *buf=data;len>=0;len--)
+		a = (a >> 8) ^ crc32_table[ (a & 0xff) ^ *buf++] ;
+
+	return a ^ 0xffffffff ;
+}
+
+bool RsDirUtil::hashFile(const std::string& f_hash, std::string& hash)
+{
+	FILE *fd;
+	int  len;
+	SHA_CTX *sha_ctx = new SHA_CTX;
+	unsigned char sha_buf[SHA_DIGEST_LENGTH];
+	unsigned char *gblBuf = new unsigned char[1024*1024*10];	// 10MB buffer.
+
+#ifdef FIM_DEBUG
+	std::cerr << "File to hash = " << f_hash << std::endl;
+#endif
+
+#ifdef WINDOWS_SYS
+        std::wstring wf_hash;
+        librs::util::ConvertUtf8ToUtf16(f_hash, wf_hash);
+        if (NULL == (fd = _wfopen(wf_hash.c_str(), L"rb")))
+            goto hashing_failed ;
+#else
+        if (NULL == (fd = fopen64(f_hash.c_str(), "rb")))
+            goto hashing_failed ;
+#endif
+
+	SHA1_Init(sha_ctx);
+	while((len = fread(gblBuf,1, 512, fd)) > 0)
+	{
+		SHA1_Update(sha_ctx, gblBuf, len);
+	}
+
+	/* reading failed for some reason */
+	if (ferror(fd))
+	{
+#ifdef FIM_DEBUG
+		std::cerr << "read error !!" << std::endl;
+#endif
+		goto hashing_failed ;
+	}
+
+	SHA1_Final(&sha_buf[0], sha_ctx);
+
+	/* TODO: Actually we should store the hash data as binary ...
+	 * but then it shouldn't be put in a string.
+	 */
+
+	{
+		std::ostringstream tmpout;
+		for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
+		{
+			tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]);
+		}
+		hash = tmpout.str();
+	}
+
+	delete sha_ctx;
+	delete[] gblBuf ;
+	fclose(fd);
+	return true;
+
+hashing_failed:
+
+	delete sha_ctx;
+	delete[] gblBuf ;
+
+	if(fd != NULL)
+		fclose(fd) ;
+
+	return false ;
+}
+
 std::string 	RsDirUtil::removeTopDir(std::string dir)
 {
 	std::string rest;

libretroshare/src/util/rsdir.h

@@ -40,12 +40,15 @@ std::string 	removeRootDir(std::string path);
 std::string     removeTopDir(std::string dir);
 std::string 	removeRootDirs(std::string path, std::string root);
 
+bool		hashFile(const std::string& full_path,std::string& hash) ;
 
 // Renames file from to file to. Files should be on the same file system.
 //	returns true if succeed, false otherwise.
 bool		renameFile(const std::string& from,const std::string& to) ;
 bool		createBackup (std::string sFilename, unsigned int nCount = 5);
 
+uint32_t rs_CRC32(const unsigned char *data,uint32_t len) ;
+
 int     	breakupDirList(std::string path,
                         	std::list<std::string> &subdirs);
 

retroshare-gui/src/gui/TransfersDialog.cpp

@@ -743,6 +743,7 @@ void TransfersDialog::insertTransfers()
 			case FT_STATE_COMPLETE:     status = tr("Complete"); break;
 			case FT_STATE_QUEUED:       status = tr("Queued"); break;
 			case FT_STATE_PAUSED:       status = tr("Paused"); break;
+			case FT_STATE_CHECKING_HASH:status = tr("Checking..."); break;
 			default:                    status = tr("Unknown"); break;
 		}