Added deferred check of chunks during file transfer. Chunk sha1 sums are requested to the sources and checked for downloaded data.

Validated chunks are shared to other peers. Force check is now very simple since it just turns all chunks into "needs checking" mode
and sums are asked to sources. Sources maintain a temporary cache of chunks. Since sums are requested sparsely, this should not 
affect the sources in terms of performance. We can still imagine precomputing and saving sha1 of chunks while hashing them.

For backward compatibility reasons, the following has been setup *temporarily* in this version:
- unvalidated chunks are still considered as already obtained, and are shared and saved
- force check has been disabled
- final file check is maintained
- in case of file fail, the old checking mode will be used.

All changes for next version are kept in the define 'USE_NEW_CHUNK_CHECKING_CODE' that will be made the default in a few weeks.
At start, I expect most chunk to stya yellow during download, until most sources are able to provide chunk hashs.




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@5019 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/util/rsdir.cc

@@ -43,6 +43,7 @@
 #include <sstream>
 
 #include <fstream>
+#include <stdexcept>
 
 #if defined(WIN32) || defined(__CYGWIN__)
 #include "util/rsstring.h"
@@ -624,18 +625,96 @@ bool RsDirUtil::getFileHash(const std::string& filepath, std::string &hash, uint
 
 	SHA1_Final(&sha_buf[0], sha_ctx);
 
-	std::ostringstream tmpout;
-	for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
-	{
-		tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]);
-	}
-	hash = tmpout.str();
+	hash = Sha1CheckSum(sha_buf).toStdString() ;
 
 	delete sha_ctx;
 	fclose(fd);
 	return true;
 }
 
+/* Function to hash, and get details of a file */
+Sha1CheckSum RsDirUtil::sha1sum(unsigned char *data, uint32_t size)
+{
+	SHA_CTX sha_ctx ;
+
+	if(SHA_DIGEST_LENGTH != 20) 
+		throw std::runtime_error("Warning: can't compute Sha1Sum with sum size != 20") ;
+
+	SHA1_Init(&sha_ctx);
+	while(size > 512)
+	{
+		SHA1_Update(&sha_ctx, data, 512);
+		data = &data[512] ;
+		size -= 512 ;
+	}
+	SHA1_Update(&sha_ctx, data, size);
+
+	unsigned char sha_buf[SHA_DIGEST_LENGTH];
+	SHA1_Final(&sha_buf[0], &sha_ctx);
+
+	return Sha1CheckSum(sha_buf) ;
+}
+
+bool Sha1CheckSum::operator==(const Sha1CheckSum& s) const
+{
+	for(int i=0;i<5;++i)
+		if(fourbytes[i] != s.fourbytes[i])
+			return false ;
+	return true ;
+}
+
+std::string Sha1CheckSum::toStdString() const
+{
+	std::ostringstream tmpout;
+
+	for(int i = 0; i < 5; i++)
+		for(int j = 0; j < 4; j++)
+			tmpout << std::setw(2) << std::setfill('0') << std::hex << ((fourbytes[i] >> (8*j)) & 0xff ) ;
+	
+	return tmpout.str() ;
+}
+
+Sha1CheckSum::Sha1CheckSum(const uint8_t *buf)
+{
+	int n=0;
+
+	for(int i = 0; i < 5; i++)
+	{
+		fourbytes[i] = 0 ;
+
+		for(int j = 0; j < 4; j++)
+			fourbytes[i] += buf[n++] << (8*j) ;
+	}
+}
+Sha1CheckSum::Sha1CheckSum(const std::string& s)
+{
+	int n=0;
+	if(s.length() != 40)
+		throw std::runtime_error("Sha1CheckSum::Sha1CheckSum: can only init from 40 chars hexadecimal string") ;
+
+	for(int i = 0; i < 5; ++i)
+	{
+		fourbytes[i] = 0 ;
+
+		for(int j = 0; j < 4; ++j)
+		{
+			for(int k=0;k<2;++k)
+			{
+				char b = s[n++] ;
+
+				if(b >= 'A' && b <= 'F')
+					fourbytes[i] += (b-'A'+10) << ((4*(1-k))+8*j) ;
+				else if(b >= 'a' && b <= 'f')
+					fourbytes[i] += (b-'a'+10) << ((4*(1-k))+8*j) ;
+				else if(b >= '0' && b <= '9')
+					fourbytes[i] += (b-'0') << ((4*(1-k))+8*j) ;
+				else
+					throw std::runtime_error("Sha1CheckSum::Sha1CheckSum: can't init from non pure hexadecimal string") ;
+			}
+		}
+	}
+}
+
 bool RsDirUtil::renameFile(const std::string& from, const std::string& to)
 {
 	int loops = 0;

libretroshare/src/util/rsdir.h

@@ -32,9 +32,10 @@
 #include <list>
 #include <stdint.h>
 
-class CRC32Map ;
 class RsThread;
 
+#include <retroshare/rstypes.h>
+
 namespace RsDirUtil {
 
 std::string 	getTopDir(const std::string&);
@@ -67,6 +68,8 @@ bool    	cleanupDirectory(const std::string& dir, const std::list<std::string> &
 bool 		hashFile(const std::string& filepath,   std::string &name, std::string &hash, uint64_t &size);
 bool 		getFileHash(const std::string& filepath,std::string &hash, uint64_t &size, RsThread *thread = NULL);
 
+Sha1CheckSum sha1sum(uint8_t *data,uint32_t size) ;
+
 
 std::wstring 	getWideTopDir(std::wstring);
 std::wstring 	getWideRootDir(std::wstring);