diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index 3255096b8..1e96a6ca9 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -42,7 +42,6 @@ #include #include #include -#include #include //*********** @@ -640,7 +639,7 @@ void FileIndexMonitor::hashFiles(const std::vector& to_hash) FileEntry fe(to_hash[i].fentries[j]) ; // copied, because hashFile updates the hash member - if (hashFile(to_hash[i].realpath, fe)) + if(RsDirUtil::hashFile(to_hash[i].realpath + "/" + to_hash[i].fentries[j].name, fe.hash)) { RsStackMutex stack(fiMutex); /**** LOCKED DIRS ****/ @@ -977,66 +976,6 @@ std::string FileIndexMonitor::locked_findRealRoot(std::string rootdir) const return realroot; } - - -bool FileIndexMonitor::hashFile(std::string fullpath, FileEntry& fent) -{ - std::string f_hash = fullpath + "/" + fent.name; - FILE *fd; - int len; - SHA_CTX *sha_ctx = new SHA_CTX; - unsigned char sha_buf[SHA_DIGEST_LENGTH]; - unsigned char gblBuf[512]; - -#ifdef FIM_DEBUG - std::cerr << "File to hash = " << f_hash << std::endl; -#endif - -#ifdef WINDOWS_SYS - std::wstring wf_hash; - librs::util::ConvertUtf8ToUtf16(f_hash, wf_hash); - if (NULL == (fd = _wfopen(wf_hash.c_str(), L"rb"))) - return false; -#else - if (NULL == (fd = fopen64(f_hash.c_str(), "rb"))) - return false; -#endif - - SHA1_Init(sha_ctx); - while((len = fread(gblBuf,1, 512, fd)) > 0) - { - SHA1_Update(sha_ctx, gblBuf, len); - } - - /* reading failed for some reason */ - if (ferror(fd)) - { -#ifdef FIM_DEBUG - std::cerr << "read error !!" << std::endl; -#endif - delete sha_ctx; - fclose(fd); - return false; - } - - SHA1_Final(&sha_buf[0], sha_ctx); - - /* TODO: Actually we should store the hash data as binary ... - * but then it shouldn't be put in a string. - */ - - std::ostringstream tmpout; - for(int i = 0; i < SHA_DIGEST_LENGTH; i++) - { - tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]); - } - fent.hash = tmpout.str(); - - delete sha_ctx; - fclose(fd); - return true; -} - int FileIndexMonitor::RequestDirDetails(std::string uid, std::string path, DirDetails &details) const { /* lock it up */ diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index 8329fabe2..0f92f6f64 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -1652,6 +1652,9 @@ bool ftController::FileDetails(std::string hash, FileInfo &info) if(it->second->mState == ftFileControl::PAUSED) info.downloadStatus = FT_STATE_PAUSED ; + if((!completed) && it->second->mTransfer->isCheckingHash()) + info.downloadStatus = FT_STATE_CHECKING_HASH ; + info.tfRate = totalRate; info.size = (it->second)->mSize; diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index ab51019ce..3ce60451d 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -2,6 +2,7 @@ #include #include #include +#include /******* * #define FILE_DEBUG 1 @@ -102,50 +103,62 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data if(!RsDiscSpace::checkForDiscSpace(RS_PARTIALS_DIRECTORY)) return false ; - RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - - if (fd == NULL) - if (!locked_initializeFileAttrs()) - return false; - - /* - * check its at the correct location - */ - if (offset + chunk_size > mSize) + bool complete = false ; { - chunk_size = mSize - offset; - std::cerr <<"Chunk Size greater than total file size, adjusting chunk " << "size " << chunk_size << std::endl; + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - } + if (fd == NULL) + if (!locked_initializeFileAttrs()) + return false; - /* - * go to the offset of the file - */ - if (0 != fseeko64(this->fd, offset, SEEK_SET)) - { - std::cerr << "ftFileCreator::addFileData() Bad fseek at offset " << offset << ", fd=" << (void*)(this->fd) << ", size=" << mSize << ", errno=" << errno << std::endl; - return 0; - } - - if (1 != fwrite(data, chunk_size, 1, this->fd)) - { - std::cerr << "ftFileCreator::addFileData() Bad fwrite." << std::endl; - std::cerr << "ERRNO: " << errno << std::endl; + /* + * check its at the correct location + */ + if (offset + chunk_size > mSize) + { + chunk_size = mSize - offset; + std::cerr <<"Chunk Size greater than total file size, adjusting chunk " << "size " << chunk_size << std::endl; - return 0; - } + } + + /* + * go to the offset of the file + */ + if (0 != fseeko64(this->fd, offset, SEEK_SET)) + { + std::cerr << "ftFileCreator::addFileData() Bad fseek at offset " << offset << ", fd=" << (void*)(this->fd) << ", size=" << mSize << ", errno=" << errno << std::endl; + return 0; + } + + if (1 != fwrite(data, chunk_size, 1, this->fd)) + { + std::cerr << "ftFileCreator::addFileData() Bad fwrite." << std::endl; + std::cerr << "ERRNO: " << errno << std::endl; + + return 0; + } #ifdef FILE_DEBUG - std::cerr << "ftFileCreator::addFileData() added Data..."; - std::cerr << std::endl; - std::cerr << " pos: " << offset; - std::cerr << std::endl; + std::cerr << "ftFileCreator::addFileData() added Data..."; + std::cerr << std::endl; + std::cerr << " pos: " << offset; + std::cerr << std::endl; #endif - /* - * Notify ftFileChunker about chunks received - */ - locked_notifyReceived(offset,chunk_size); + /* + * Notify ftFileChunker about chunks received + */ + locked_notifyReceived(offset,chunk_size); + complete = chunkMap.isComplete(); + } + if(complete) + { +#ifdef FILE_DEBUG + std::cerr << "ftFileCreator::addFileData() File is complete: closing" << std::endl ; +#endif + closeFile(); + } + /* * FIXME HANDLE COMPLETION HERE - Any better way? */ @@ -451,3 +464,17 @@ bool ftFileCreator::finished() return chunkMap.isComplete() ; } +bool ftFileCreator::hashReceivedData(std::string& hash) +{ + RsStackMutex stack(ftcMutex) ; + + return RsDirUtil::hashFile(file_name,hash) ; +} + +#include +bool ftFileCreator::crossCheckChunkMap(const CRC32Map& ref) +{ + // still to be implemented. + throw std::runtime_error(std::string("Unimplemented function ") + __PRETTY_FUNCTION__) ; + return true ; +} diff --git a/libretroshare/src/ft/ftfilecreator.h b/libretroshare/src/ft/ftfilecreator.h index d44f02a21..a5a60affe 100644 --- a/libretroshare/src/ft/ftfilecreator.h +++ b/libretroshare/src/ft/ftfilecreator.h @@ -57,6 +57,17 @@ class ftFileCreator: public ftFileProvider void setChunkStrategy(FileChunksInfo::ChunkStrategy s) ; FileChunksInfo::ChunkStrategy getChunkStrategy() ; + // Computes a sha1sum of the partial file, to check that the data is overall consistent. + // + bool hashReceivedData(std::string& hash) ; + + // Computes a CRC32 map of all chunks, for comparison with reference, and re-starting invalid chunks. + // + bool CRC32ReceivedData(CRC32Map& crc_map) ; + + // Checks the CRC32 of all chunks against the given CRC32 map. Re-flag the bad chunks as being void. + // + bool crossCheckChunkMap(const CRC32Map& crc_map) ; /* * creation functions for FileCreator */ diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index b4e600903..35c5da9f0 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -58,8 +58,12 @@ const int32_t FT_TM_FAST_RTT = 1.0; const int32_t FT_TM_STD_RTT = 5.0; const int32_t FT_TM_SLOW_RTT = 9.0; +#define FT_TM_FLAG_DOWNLOADING 0 +#define FT_TM_FLAG_CHECKING 3 +#define FT_TM_FLAG_CHECKING 3 + ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm, ftController *c) - :mFileCreator(fc), mMultiplexor(dm), mFtController(c), mFlag(0) + :mFileCreator(fc), mMultiplexor(dm), mFtController(c), mFlag(FT_TM_FLAG_DOWNLOADING) { RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ @@ -376,31 +380,37 @@ bool ftTransferModule::queryInactive() { /* NB: Not sure about this lock... might cause deadlock. */ - RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ #ifdef FT_DEBUG - std::cerr << "ftTransferModule::queryInactive()" << std::endl; + std::cerr << "ftTransferModule::queryInactive()" << std::endl; #endif - if (mFileStatus.stat == ftFileStatus::PQIFILE_INIT) + if (mFileStatus.stat == ftFileStatus::PQIFILE_INIT) mFileStatus.stat = ftFileStatus::PQIFILE_DOWNLOADING; - if (mFileStatus.stat != ftFileStatus::PQIFILE_DOWNLOADING) - { + if (mFileStatus.stat == ftFileStatus::PQIFILE_CHECKING) + return false ; + + if (mFileStatus.stat != ftFileStatus::PQIFILE_DOWNLOADING) + { if (mFileStatus.stat == ftFileStatus::PQIFILE_FAIL_CANCEL) mFlag = 2; //file canceled by user return false; } - std::map::iterator mit; - for(mit = mFileSources.begin(); mit != mFileSources.end(); mit++) - { + std::map::iterator mit; + for(mit = mFileSources.begin(); mit != mFileSources.end(); mit++) + { locked_tickPeerTransfer(mit->second); } if(mFileCreator->finished()) // transfer is complete - mFlag = 1; + { + mFileStatus.stat = ftFileStatus::PQIFILE_CHECKING ; + mFlag = 3; + } - return true; + return true; } bool ftTransferModule::cancelTransfer() @@ -411,6 +421,12 @@ bool ftTransferModule::cancelTransfer() return 1; } +bool ftTransferModule::cancelFileTransferUpward() +{ + if (mFtController) + mFtController->FileCancel(mHash); + return true; +} bool ftTransferModule::completeFileTransfer() { #ifdef FT_DEBUG @@ -463,6 +479,14 @@ int ftTransferModule::tick() break; case 2: //file transfer canceled break; + case 3: + // Check if file hash matches the hashed data + checkFile() ; + break ; + case 4: + // File is waiting for CRC32 map. Check if received, and re-set matched chunks + checkCRC() ; + break ; default: break; } @@ -470,6 +494,67 @@ int ftTransferModule::tick() return 0; } +bool ftTransferModule::isCheckingHash() +{ + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ + return mFlag == 3 ; +} +bool ftTransferModule::checkFile() +{ + { + RsStackMutex stack(tfMtx); /******* STACK LOCKED ******/ +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::checkFile(): checking File " << mHash << std::endl ; +#endif + std::string hash_check ; + + if(!mFileCreator->hashReceivedData(hash_check)) + { + std::cerr << "ftTransferModule::checkFile(): Impossible to hash received file " << mHash << " for check. What is happenning?" << std::endl ; + return false ; + } + + if(hash_check == mHash) + { + mFlag = 1 ; // Transfer is complete. +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::checkFile(): File verification complete ! Setting mFlag to 1" << std::endl ; +#endif + return true ; + } + + mFlag = 4 ; // Ask for CRC map. But for now, cancel file transfer. + mFileStatus.stat = ftFileStatus::PQIFILE_FAIL_CANCEL ; + } + cancelFileTransferUpward() ; + std::cerr << "(EE) ftTransferModule::checkFile(): File verification failed for hash " << mHash << "! Asking for CRC map. mFlag=4. For now: cancelling file transfer." << std::endl ; + //askForCRCMap() ; + + return false ; +} + +bool ftTransferModule::checkCRC() +{ +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::checkCRC(): looking for CRC32 map." << std::endl ; +#endif + + if(_crc32map.empty()) + return false ; + +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::checkCRC(): got a CRC32 map. Matching chunks..." << std::endl ; +#endif + + mFileCreator->crossCheckChunkMap(_crc32map) ; + mFlag = 0 ; + +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::checkCRC(): Done. Restarting download." << std::endl ; +#endif + return true ; +} + void ftTransferModule::adjustSpeed() { diff --git a/libretroshare/src/ft/fttransfermodule.h b/libretroshare/src/ft/fttransfermodule.h index 4a8a7f46b..2d80e290e 100644 --- a/libretroshare/src/ft/fttransfermodule.h +++ b/libretroshare/src/ft/fttransfermodule.h @@ -106,6 +106,7 @@ public: PQIFILE_NOT_ONLINE, PQIFILE_DOWNLOADING, PQIFILE_COMPLETE, + PQIFILE_CHECKING, PQIFILE_FAIL, PQIFILE_FAIL_CANCEL, PQIFILE_FAIL_NOT_AVAIL, @@ -138,7 +139,9 @@ public: bool getPeerState(std::string peerId,uint32_t &state,uint32_t &tfRate); uint32_t getDataRate(std::string peerId); bool cancelTransfer(); + bool cancelFileTransferUpward(); bool completeFileTransfer(); + bool isCheckingHash() ; //interface to multiplex module bool recvFileData(std::string peerId, uint64_t offset, @@ -164,6 +167,8 @@ private: bool locked_recvPeerData(peerInfo &info, uint64_t offset, uint32_t chunk_size, void *data); + bool checkFile() ; + bool checkCRC() ; /* These have independent Mutexes / are const locally (no Mutex protection)*/ ftFileCreator *mFileCreator; @@ -178,10 +183,12 @@ private: std::list mOnlinePeers; std::map mFileSources; - uint16_t mFlag; //2:file canceled, 1:transfer complete, 0: not complete + uint16_t mFlag; //2:file canceled, 1:transfer complete, 0: not complete, 3: checking hash, 4: checking chunks double desiredRate; double actualRate; + CRC32Map _crc32map ; + ftFileStatus mFileStatus; //used for pause/resume file transfer }; diff --git a/libretroshare/src/rsiface/rstypes.h b/libretroshare/src/rsiface/rstypes.h index 9bd8d2e22..91debf9b4 100644 --- a/libretroshare/src/rsiface/rstypes.h +++ b/libretroshare/src/rsiface/rstypes.h @@ -39,13 +39,14 @@ typedef std::string RsChanId; typedef std::string RsMsgId; typedef std::string RsAuthId; -const uint32_t FT_STATE_FAILED = 0x0000 ; -const uint32_t FT_STATE_OKAY = 0x0001 ; -const uint32_t FT_STATE_WAITING = 0x0002 ; -const uint32_t FT_STATE_DOWNLOADING = 0x0003 ; -const uint32_t FT_STATE_COMPLETE = 0x0004 ; -const uint32_t FT_STATE_QUEUED = 0x0005 ; -const uint32_t FT_STATE_PAUSED = 0x0006 ; +const uint32_t FT_STATE_FAILED = 0x0000 ; +const uint32_t FT_STATE_OKAY = 0x0001 ; +const uint32_t FT_STATE_WAITING = 0x0002 ; +const uint32_t FT_STATE_DOWNLOADING = 0x0003 ; +const uint32_t FT_STATE_COMPLETE = 0x0004 ; +const uint32_t FT_STATE_QUEUED = 0x0005 ; +const uint32_t FT_STATE_PAUSED = 0x0006 ; +const uint32_t FT_STATE_CHECKING_HASH = 0x0007 ; // These constants are used by RsDiscSpace // @@ -338,6 +339,23 @@ class CompressedChunkMap std::vector _map ; }; +class CRC32Map +{ + public: + // Build from a file. + // + CRC32Map(const std::string& fname,uint32_t chunk_size) ; + CRC32Map() {} + + // Compares two maps and returns the valid chunks in a compressed chunk map. + // + friend CompressedChunkMap compare(const CRC32Map& crc1,const CRC32Map& crc2) ; + + bool empty() const { return _map.empty() ; } + private: + std::vector _map ; +}; + /* class which encapsulates download details */ class DwlDetails { public: diff --git a/libretroshare/src/util/rsdir.cc b/libretroshare/src/util/rsdir.cc index b16035269..44185de07 100644 --- a/libretroshare/src/util/rsdir.cc +++ b/libretroshare/src/util/rsdir.cc @@ -36,6 +36,9 @@ #include #include #include +#include +#include +#include #include @@ -74,6 +77,121 @@ std::string RsDirUtil::getTopDir(std::string dir) return top; } +class CRC32Table +{ + public: + inline uint32_t operator[](unsigned char i) const { return _data[i] ; } + + CRC32Table() + { + _data = new uint32_t[256] ; + uint32_t polynmial = 0x04c11db7 ; + + for(uint32_t i=0;i<256;++i) + { + uint32_t a = reflect(i,8)<<24 ; + + for(uint32_t j=0;j<8;++j) + a = (a << 1)^ ( (a&(1<<31))?polynmial:0) ; + + _data[i] = reflect(a,32) ; + } + } + // Swap bits 0-7, 1-6, etc. + // + uint32_t reflect(uint32_t ref,unsigned char ch) + { + uint32_t val = 0 ; + for(int i=1;i>=1) + if(ref & 1) + val |= (1 << (ch-i)) ; + return val ; + } + ~CRC32Table() { delete[] _data ; } + + private: + uint32_t *_data ; +}; + +uint32_t rs_CRC32(const unsigned char *data,uint32_t len) +{ + static const CRC32Table crc32_table ; + uint32_t a = 0xffffffff ; + + for(const unsigned char *buf=data;len>=0;len--) + a = (a >> 8) ^ crc32_table[ (a & 0xff) ^ *buf++] ; + + return a ^ 0xffffffff ; +} + +bool RsDirUtil::hashFile(const std::string& f_hash, std::string& hash) +{ + FILE *fd; + int len; + SHA_CTX *sha_ctx = new SHA_CTX; + unsigned char sha_buf[SHA_DIGEST_LENGTH]; + unsigned char *gblBuf = new unsigned char[1024*1024*10]; // 10MB buffer. + +#ifdef FIM_DEBUG + std::cerr << "File to hash = " << f_hash << std::endl; +#endif + +#ifdef WINDOWS_SYS + std::wstring wf_hash; + librs::util::ConvertUtf8ToUtf16(f_hash, wf_hash); + if (NULL == (fd = _wfopen(wf_hash.c_str(), L"rb"))) + goto hashing_failed ; +#else + if (NULL == (fd = fopen64(f_hash.c_str(), "rb"))) + goto hashing_failed ; +#endif + + SHA1_Init(sha_ctx); + while((len = fread(gblBuf,1, 512, fd)) > 0) + { + SHA1_Update(sha_ctx, gblBuf, len); + } + + /* reading failed for some reason */ + if (ferror(fd)) + { +#ifdef FIM_DEBUG + std::cerr << "read error !!" << std::endl; +#endif + goto hashing_failed ; + } + + SHA1_Final(&sha_buf[0], sha_ctx); + + /* TODO: Actually we should store the hash data as binary ... + * but then it shouldn't be put in a string. + */ + + { + std::ostringstream tmpout; + for(int i = 0; i < SHA_DIGEST_LENGTH; i++) + { + tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]); + } + hash = tmpout.str(); + } + + delete sha_ctx; + delete[] gblBuf ; + fclose(fd); + return true; + +hashing_failed: + + delete sha_ctx; + delete[] gblBuf ; + + if(fd != NULL) + fclose(fd) ; + + return false ; +} + std::string RsDirUtil::removeTopDir(std::string dir) { std::string rest; diff --git a/libretroshare/src/util/rsdir.h b/libretroshare/src/util/rsdir.h index ffc824bda..3ced06a38 100644 --- a/libretroshare/src/util/rsdir.h +++ b/libretroshare/src/util/rsdir.h @@ -40,12 +40,15 @@ std::string removeRootDir(std::string path); std::string removeTopDir(std::string dir); std::string removeRootDirs(std::string path, std::string root); +bool hashFile(const std::string& full_path,std::string& hash) ; // Renames file from to file to. Files should be on the same file system. // returns true if succeed, false otherwise. bool renameFile(const std::string& from,const std::string& to) ; bool createBackup (std::string sFilename, unsigned int nCount = 5); +uint32_t rs_CRC32(const unsigned char *data,uint32_t len) ; + int breakupDirList(std::string path, std::list &subdirs); diff --git a/retroshare-gui/src/gui/TransfersDialog.cpp b/retroshare-gui/src/gui/TransfersDialog.cpp index 755eee80e..03e8428d6 100644 --- a/retroshare-gui/src/gui/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/TransfersDialog.cpp @@ -743,6 +743,7 @@ void TransfersDialog::insertTransfers() case FT_STATE_COMPLETE: status = tr("Complete"); break; case FT_STATE_QUEUED: status = tr("Queued"); break; case FT_STATE_PAUSED: status = tr("Paused"); break; + case FT_STATE_CHECKING_HASH:status = tr("Checking..."); break; default: status = tr("Unknown"); break; }