Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

- implemented post-download hash re-check. For now, if the hash does not match, the download is canceled, but in the near future, per-chunk comparison wil occur

Browse Source 
.
- corrected a bug that caused file copy error: a closeFile() was missing when the file is complete. Because of delays in fwrite, the file would not be always co
mplete, nor exist at all for small files (e.g. cache files), which in the later case caused the copy error.

Warning: needs a make clean in libretroshare to recompile.



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@3261 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
csoler 2010-07-06 05:04:11 +00:00
parent 225781aa51
commit 458a8faf70
10 changed files with 329 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

63
libretroshare/src/dbase/fimonitor.cc
View File

@ -42,7 +42,6 @@
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#include <openssl/sha.h>
#include <stdio.h>
//***********
@ -640,7 +639,7 @@ void FileIndexMonitor::hashFiles(const std::vector<DirContentToHash>& to_hash)
FileEntry fe(to_hash[i].fentries[j]) ; // copied, because hashFile updates the hash member
if (hashFile(to_hash[i].realpath, fe))
if(RsDirUtil::hashFile(to_hash[i].realpath + "/" + to_hash[i].fentries[j].name, fe.hash))
{
RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/
@ -977,66 +976,6 @@ std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const
return realroot;
}
bool FileIndexMonitor::hashFile(std::string fullpath, FileEntry& fent)
{
std::string f_hash = fullpath + "/" + fent.name;
FILE *fd;
int len;
SHA_CTX *sha_ctx = new SHA_CTX;
unsigned char sha_buf[SHA_DIGEST_LENGTH];
unsigned char gblBuf[512];
#ifdef FIM_DEBUG
std::cerr << "File to hash = " << f_hash << std::endl;
#endif
#ifdef WINDOWS_SYS
std::wstring wf_hash;
librs::util::ConvertUtf8ToUtf16(f_hash, wf_hash);
if (NULL == (fd = _wfopen(wf_hash.c_str(), L"rb")))
return false;
#else
if (NULL == (fd = fopen64(f_hash.c_str(), "rb")))
return false;
#endif
SHA1_Init(sha_ctx);
while((len = fread(gblBuf,1, 512, fd)) > 0)
{
SHA1_Update(sha_ctx, gblBuf, len);
}
/* reading failed for some reason */
if (ferror(fd))
{
#ifdef FIM_DEBUG
std::cerr << "read error !!" << std::endl;
#endif
delete sha_ctx;
fclose(fd);
return false;
}
SHA1_Final(&sha_buf[0], sha_ctx);
/* TODO: Actually we should store the hash data as binary ...
* but then it shouldn't be put in a string.
*/
std::ostringstream tmpout;
for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
{
tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]);
}
fent.hash = tmpout.str();
delete sha_ctx;
fclose(fd);
return true;
}
int FileIndexMonitor::RequestDirDetails(std::string uid, std::string path, DirDetails &details) const
{
/* lock it up */

3
libretroshare/src/ft/ftcontroller.cc
View File

@ -1652,6 +1652,9 @@ bool ftController::FileDetails(std::string hash, FileInfo &info)
if(it->second->mState == ftFileControl::PAUSED)
info.downloadStatus = FT_STATE_PAUSED ;
if((!completed) && it->second->mTransfer->isCheckingHash())
info.downloadStatus = FT_STATE_CHECKING_HASH ;
info.tfRate = totalRate;
info.size = (it->second)->mSize;

97
libretroshare/src/ft/ftfilecreator.cc
View File

@ -2,6 +2,7 @@
#include <errno.h>
#include <stdio.h>
#include <util/rsdiscspace.h>
#include <util/rsdir.h>
/*******
* #define FILE_DEBUG 1
@ -102,49 +103,61 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data
if(!RsDiscSpace::checkForDiscSpace(RS_PARTIALS_DIRECTORY))
return false ;
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
if (fd == NULL)
if (!locked_initializeFileAttrs())
return false;
/*
* check its at the correct location
*/
if (offset + chunk_size > mSize)
bool complete = false ;
{
chunk_size = mSize - offset;
std::cerr <<"Chunk Size greater than total file size, adjusting chunk " << "size " << chunk_size << std::endl;
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
}
if (fd == NULL)
if (!locked_initializeFileAttrs())
return false;
/*
* go to the offset of the file
*/
if (0 != fseeko64(this->fd, offset, SEEK_SET))
{
std::cerr << "ftFileCreator::addFileData() Bad fseek at offset " << offset << ", fd=" << (void*)(this->fd) << ", size=" << mSize << ", errno=" << errno << std::endl;
return 0;
}
/*
* check its at the correct location
*/
if (offset + chunk_size > mSize)
{
chunk_size = mSize - offset;
std::cerr <<"Chunk Size greater than total file size, adjusting chunk " << "size " << chunk_size << std::endl;
if (1 != fwrite(data, chunk_size, 1, this->fd))
{
std::cerr << "ftFileCreator::addFileData() Bad fwrite." << std::endl;
std::cerr << "ERRNO: " << errno << std::endl;
}
return 0;
}
/*
* go to the offset of the file
*/
if (0 != fseeko64(this->fd, offset, SEEK_SET))
{
std::cerr << "ftFileCreator::addFileData() Bad fseek at offset " << offset << ", fd=" << (void*)(this->fd) << ", size=" << mSize << ", errno=" << errno << std::endl;
return 0;
}
if (1 != fwrite(data, chunk_size, 1, this->fd))
{
std::cerr << "ftFileCreator::addFileData() Bad fwrite." << std::endl;
std::cerr << "ERRNO: " << errno << std::endl;
return 0;
}
#ifdef FILE_DEBUG
std::cerr << "ftFileCreator::addFileData() added Data...";
std::cerr << std::endl;
std::cerr << " pos: " << offset;
std::cerr << std::endl;
std::cerr << "ftFileCreator::addFileData() added Data...";
std::cerr << std::endl;
std::cerr << " pos: " << offset;
std::cerr << std::endl;
#endif
/*
* Notify ftFileChunker about chunks received
*/
locked_notifyReceived(offset,chunk_size);
/*
* Notify ftFileChunker about chunks received
*/
locked_notifyReceived(offset,chunk_size);
complete = chunkMap.isComplete();
}
if(complete)
{
#ifdef FILE_DEBUG
std::cerr << "ftFileCreator::addFileData() File is complete: closing" << std::endl ;
#endif
closeFile();
}
/*
* FIXME HANDLE COMPLETION HERE - Any better way?
@ -451,3 +464,17 @@ bool ftFileCreator::finished()
return chunkMap.isComplete() ;
}
bool ftFileCreator::hashReceivedData(std::string& hash)
{
RsStackMutex stack(ftcMutex) ;
return RsDirUtil::hashFile(file_name,hash) ;
}
#include <stdexcept>
bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref)
{
// still to be implemented.
throw std::runtime_error(std::string("Unimplemented function ") + __PRETTY_FUNCTION__) ;
return true ;
}

11
libretroshare/src/ft/ftfilecreator.h
View File

@ -57,6 +57,17 @@ class ftFileCreator: public ftFileProvider
void setChunkStrategy(FileChunksInfo::ChunkStrategy s) ;
FileChunksInfo::ChunkStrategy getChunkStrategy() ;
// Computes a sha1sum of the partial file, to check that the data is overall consistent.
//
bool hashReceivedData(std::string& hash) ;
// Computes a CRC32 map of all chunks, for comparison with reference, and re-starting invalid chunks.
//
bool CRC32ReceivedData(CRC32Map& crc_map) ;
// Checks the CRC32 of all chunks against the given CRC32 map. Re-flag the bad chunks as being void.
//
bool crossCheckChunkMap(const CRC32Map& crc_map) ;
/*
* creation functions for FileCreator
*/

107
libretroshare/src/ft/fttransfermodule.cc
View File

@ -58,8 +58,12 @@ const int32_t FT_TM_FAST_RTT = 1.0;
const int32_t FT_TM_STD_RTT = 5.0;
const int32_t FT_TM_SLOW_RTT = 9.0;
#define FT_TM_FLAG_DOWNLOADING 0
#define FT_TM_FLAG_CHECKING 3
#define FT_TM_FLAG_CHECKING 3
ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm, ftController *c)
:mFileCreator(fc), mMultiplexor(dm), mFtController(c), mFlag(0)
:mFileCreator(fc), mMultiplexor(dm), mFtController(c), mFlag(FT_TM_FLAG_DOWNLOADING)
{
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
@ -376,31 +380,37 @@ bool ftTransferModule::queryInactive()
{
/* NB: Not sure about this lock... might cause deadlock.
*/
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::queryInactive()" << std::endl;
std::cerr << "ftTransferModule::queryInactive()" << std::endl;
#endif
if (mFileStatus.stat == ftFileStatus::PQIFILE_INIT)
if (mFileStatus.stat == ftFileStatus::PQIFILE_INIT)
mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING;
if (mFileStatus.stat != ftFileStatus::PQIFILE_DOWNLOADING)
{
if (mFileStatus.stat == ftFileStatus::PQIFILE_CHECKING)
return false ;
if (mFileStatus.stat != ftFileStatus::PQIFILE_DOWNLOADING)
{
if (mFileStatus.stat == ftFileStatus::PQIFILE_FAIL_CANCEL)
mFlag = 2; //file canceled by user
return false;
}
std::map<std::string,peerInfo>::iterator mit;
for(mit = mFileSources.begin(); mit != mFileSources.end(); mit++)
{
std::map<std::string,peerInfo>::iterator mit;
for(mit = mFileSources.begin(); mit != mFileSources.end(); mit++)
{
locked_tickPeerTransfer(mit->second);
}
if(mFileCreator->finished()) // transfer is complete
mFlag = 1;
{
mFileStatus.stat = ftFileStatus::PQIFILE_CHECKING ;
mFlag = 3;
}
return true;
return true;
}
bool ftTransferModule::cancelTransfer()
@ -411,6 +421,12 @@ bool ftTransferModule::cancelTransfer()
return 1;
}
bool ftTransferModule::cancelFileTransferUpward()
{
if (mFtController)
mFtController->FileCancel(mHash);
return true;
}
bool ftTransferModule::completeFileTransfer()
{
#ifdef FT_DEBUG
@ -463,6 +479,14 @@ int ftTransferModule::tick()
break;
case 2: //file transfer canceled
break;
case 3:
// Check if file hash matches the hashed data
checkFile() ;
break ;
case 4:
// File is waiting for CRC32 map. Check if received, and re-set matched chunks
checkCRC() ;
break ;
default:
break;
}
@ -470,6 +494,67 @@ int ftTransferModule::tick()
return 0;
}
bool ftTransferModule::isCheckingHash()
{
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
return mFlag == 3 ;
}
bool ftTransferModule::checkFile()
{
{
RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkFile(): checking File " << mHash << std::endl ;
#endif
std::string hash_check ;
if(!mFileCreator->hashReceivedData(hash_check))
{
std::cerr << "ftTransferModule::checkFile(): Impossible to hash received file " << mHash << " for check. What is happenning?" << std::endl ;
return false ;
}
if(hash_check == mHash)
{
mFlag = 1 ; // Transfer is complete.
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkFile(): File verification complete ! Setting mFlag to 1" << std::endl ;
#endif
return true ;
}
mFlag = 4 ; // Ask for CRC map. But for now, cancel file transfer.
mFileStatus.stat = ftFileStatus::PQIFILE_FAIL_CANCEL ;
}
cancelFileTransferUpward() ;
std::cerr << "(EE) ftTransferModule::checkFile(): File verification failed for hash " << mHash << "! Asking for CRC map. mFlag=4. For now: cancelling file transfer." << std::endl ;
//askForCRCMap() ;
return false ;
}
bool ftTransferModule::checkCRC()
{
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): looking for CRC32 map." << std::endl ;
#endif
if(_crc32map.empty())
return false ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): got a CRC32 map. Matching chunks..." << std::endl ;
#endif
mFileCreator->crossCheckChunkMap(_crc32map) ;
mFlag = 0 ;
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::checkCRC(): Done. Restarting download." << std::endl ;
#endif
return true ;
}
void ftTransferModule::adjustSpeed()
{

9
libretroshare/src/ft/fttransfermodule.h
View File

@ -106,6 +106,7 @@ public:
PQIFILE_NOT_ONLINE,
PQIFILE_DOWNLOADING,
PQIFILE_COMPLETE,
PQIFILE_CHECKING,
PQIFILE_FAIL,
PQIFILE_FAIL_CANCEL,
PQIFILE_FAIL_NOT_AVAIL,
@ -138,7 +139,9 @@ public:
bool getPeerState(std::string peerId,uint32_t &state,uint32_t &tfRate);
uint32_t getDataRate(std::string peerId);
bool cancelTransfer();
bool cancelFileTransferUpward();
bool completeFileTransfer();
bool isCheckingHash() ;
//interface to multiplex module
bool recvFileData(std::string peerId, uint64_t offset,
@ -164,6 +167,8 @@ private:
bool locked_recvPeerData(peerInfo &info, uint64_t offset,
uint32_t chunk_size, void *data);
bool checkFile() ;
bool checkCRC() ;
/* These have independent Mutexes / are const locally (no Mutex protection)*/
ftFileCreator *mFileCreator;
@ -178,10 +183,12 @@ private:
std::list<std::string> mOnlinePeers;
std::map<std::string,peerInfo> mFileSources;
uint16_t mFlag; //2:file canceled, 1:transfer complete, 0: not complete
uint16_t mFlag; //2:file canceled, 1:transfer complete, 0: not complete, 3: checking hash, 4: checking chunks
double desiredRate;
double actualRate;
CRC32Map _crc32map ;
ftFileStatus mFileStatus; //used for pause/resume file transfer
};

32
libretroshare/src/rsiface/rstypes.h
View File

@ -39,13 +39,14 @@ typedef std::string RsChanId;
typedef std::string RsMsgId;
typedef std::string RsAuthId;
const uint32_t FT_STATE_FAILED = 0x0000 ;
const uint32_t FT_STATE_OKAY = 0x0001 ;
const uint32_t FT_STATE_WAITING = 0x0002 ;
const uint32_t FT_STATE_DOWNLOADING = 0x0003 ;
const uint32_t FT_STATE_COMPLETE = 0x0004 ;
const uint32_t FT_STATE_QUEUED = 0x0005 ;
const uint32_t FT_STATE_PAUSED = 0x0006 ;
const uint32_t FT_STATE_FAILED = 0x0000 ;
const uint32_t FT_STATE_OKAY = 0x0001 ;
const uint32_t FT_STATE_WAITING = 0x0002 ;
const uint32_t FT_STATE_DOWNLOADING = 0x0003 ;
const uint32_t FT_STATE_COMPLETE = 0x0004 ;
const uint32_t FT_STATE_QUEUED = 0x0005 ;
const uint32_t FT_STATE_PAUSED = 0x0006 ;
const uint32_t FT_STATE_CHECKING_HASH = 0x0007 ;
// These constants are used by RsDiscSpace
//
@ -338,6 +339,23 @@ class CompressedChunkMap
std::vector<uint32_t> _map ;
};
class CRC32Map
{
public:
// Build from a file.
//
CRC32Map(const std::string& fname,uint32_t chunk_size) ;
CRC32Map() {}
// Compares two maps and returns the valid chunks in a compressed chunk map.
//
friend CompressedChunkMap compare(const CRC32Map& crc1,const CRC32Map& crc2) ;
bool empty() const { return _map.empty() ; }
private:
std::vector<uint32_t> _map ;
};
/* class which encapsulates download details */
class DwlDetails {
public:

118
libretroshare/src/util/rsdir.cc
View File

@ -36,6 +36,9 @@
#include <algorithm>
#include <stdio.h>
#include <dirent.h>
#include <openssl/sha.h>
#include <iomanip>
#include <sstream>
#include <fstream>
@ -74,6 +77,121 @@ std::string RsDirUtil::getTopDir(std::string dir)
return top;
}
class CRC32Table
{
public:
inline uint32_t operator[](unsigned char i) const { return _data[i] ; }
CRC32Table()
{
_data = new uint32_t[256] ;
uint32_t polynmial = 0x04c11db7 ;
for(uint32_t i=0;i<256;++i)
{
uint32_t a = reflect(i,8)<<24 ;
for(uint32_t j=0;j<8;++j)
a = (a << 1)^ ( (a&(1<<31))?polynmial:0) ;
_data[i] = reflect(a,32) ;
}
}
// Swap bits 0-7, 1-6, etc.
//
uint32_t reflect(uint32_t ref,unsigned char ch)
{
uint32_t val = 0 ;
for(int i=1;i<ch+1;i++,ref>>=1)
if(ref & 1)
val |= (1 << (ch-i)) ;
return val ;
}
~CRC32Table() { delete[] _data ; }
private:
uint32_t *_data ;
};
uint32_t rs_CRC32(const unsigned char *data,uint32_t len)
{
static const CRC32Table crc32_table ;
uint32_t a = 0xffffffff ;
for(const unsigned char *buf=data;len>=0;len--)
a = (a >> 8) ^ crc32_table[ (a & 0xff) ^ *buf++] ;
return a ^ 0xffffffff ;
}
bool RsDirUtil::hashFile(const std::string& f_hash, std::string& hash)
{
FILE *fd;
int len;
SHA_CTX *sha_ctx = new SHA_CTX;
unsigned char sha_buf[SHA_DIGEST_LENGTH];
unsigned char *gblBuf = new unsigned char[1024*1024*10]; // 10MB buffer.
#ifdef FIM_DEBUG
std::cerr << "File to hash = " << f_hash << std::endl;
#endif
#ifdef WINDOWS_SYS
std::wstring wf_hash;
librs::util::ConvertUtf8ToUtf16(f_hash, wf_hash);
if (NULL == (fd = _wfopen(wf_hash.c_str(), L"rb")))
goto hashing_failed ;
#else
if (NULL == (fd = fopen64(f_hash.c_str(), "rb")))
goto hashing_failed ;
#endif
SHA1_Init(sha_ctx);
while((len = fread(gblBuf,1, 512, fd)) > 0)
{
SHA1_Update(sha_ctx, gblBuf, len);
}
/* reading failed for some reason */
if (ferror(fd))
{
#ifdef FIM_DEBUG
std::cerr << "read error !!" << std::endl;
#endif
goto hashing_failed ;
}
SHA1_Final(&sha_buf[0], sha_ctx);
/* TODO: Actually we should store the hash data as binary ...
* but then it shouldn't be put in a string.
*/
{
std::ostringstream tmpout;
for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
{
tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]);
}
hash = tmpout.str();
}
delete sha_ctx;
delete[] gblBuf ;
fclose(fd);
return true;
hashing_failed:
delete sha_ctx;
delete[] gblBuf ;
if(fd != NULL)
fclose(fd) ;
return false ;
}
std::string RsDirUtil::removeTopDir(std::string dir)
{
std::string rest;

3
libretroshare/src/util/rsdir.h
View File

@ -40,12 +40,15 @@ std::string removeRootDir(std::string path);
std::string removeTopDir(std::string dir);
std::string removeRootDirs(std::string path, std::string root);
bool hashFile(const std::string& full_path,std::string& hash) ;
// Renames file from to file to. Files should be on the same file system.
// returns true if succeed, false otherwise.
bool renameFile(const std::string& from,const std::string& to) ;
bool createBackup (std::string sFilename, unsigned int nCount = 5);
uint32_t rs_CRC32(const unsigned char *data,uint32_t len) ;
int breakupDirList(std::string path,
std::list<std::string> &subdirs);

1
retroshare-gui/src/gui/TransfersDialog.cpp
View File

@ -743,6 +743,7 @@ void TransfersDialog::insertTransfers()
case FT_STATE_COMPLETE: status = tr("Complete"); break;
case FT_STATE_QUEUED: status = tr("Queued"); break;
case FT_STATE_PAUSED: status = tr("Paused"); break;
case FT_STATE_CHECKING_HASH:status = tr("Checking..."); break;
default: status = tr("Unknown"); break;
}