diff --git a/libretroshare/src/file_sharing/directory_storage.h b/libretroshare/src/file_sharing/directory_storage.h index a3a084612..9e7872611 100644 --- a/libretroshare/src/file_sharing/directory_storage.h +++ b/libretroshare/src/file_sharing/directory_storage.h @@ -7,15 +7,6 @@ #include "retroshare/rsids.h" #include "retroshare/rsfiles.h" -static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; - -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ; -static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ; -static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ; -static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ; -static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ; - #define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } class InternalFileHierarchyStorage ; diff --git a/libretroshare/src/file_sharing/filelist_io.cc b/libretroshare/src/file_sharing/filelist_io.cc new file mode 100644 index 000000000..e8e1207f7 --- /dev/null +++ b/libretroshare/src/file_sharing/filelist_io.cc @@ -0,0 +1,2 @@ +#include "filelist_io.h" + diff --git a/libretroshare/src/file_sharing/filelist_io.h b/libretroshare/src/file_sharing/filelist_io.h new file mode 100644 index 000000000..d8f5f0d10 --- /dev/null +++ b/libretroshare/src/file_sharing/filelist_io.h @@ -0,0 +1,58 @@ +#pragma once + +#include + +// This file implements load/save of various fields used for file lists and directory content. +// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers. + +static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ; + +static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ; +static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ; +static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ; +static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ; +static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ; +static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ; +static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ; + +class FileListIO +{ +public: + template + static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val) + { + if(!checkSectionSize(buff,buff_size,offset,sizeof(T))) + return false; + + if(!writeSectionHeader(buff,buff_size,offset,section_tag,sizeof(T))) + return false; + + memcpy(&buff[offset],reinterpret_cast(&val),sizeof(T)) ; + offset += sizeof(T) ; + + return true; + } + +private: + static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S) + { + if(offset + S > buff_size) + { + buff = (unsigned char *)realloc(buff,offset + S) ; + buff_size = offset + S ; + + if(!buff) + return false ; + } + return true ; + } + + static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint8_t section_tag,uint32_t S) + { + buff[offset++] = section_tag ; + if(!write125Size(buff,offset,buff_size,S)) return false ; + + return true; + } +}; diff --git a/libretroshare/src/file_sharing/hash_cache.cc b/libretroshare/src/file_sharing/hash_cache.cc index 32401ca4b..c86e6e8cc 100644 --- a/libretroshare/src/file_sharing/hash_cache.cc +++ b/libretroshare/src/file_sharing/hash_cache.cc @@ -1,13 +1,19 @@ #include "util/rsdir.h" #include "rsserver/p3face.h" +#include "pqi/authssl.h" #include "hash_cache.h" +#include "filelist_io.h" #define HASHSTORAGE_DEBUG 1 +static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000; +static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000; + HashStorage::HashStorage(const std::string& save_file_name) : mFilePath(save_file_name), mHashMtx("Hash Storage mutex") { mRunning = false ; + load() ; } void HashStorage::data_tick() @@ -28,9 +34,15 @@ void HashStorage::data_tick() RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ; - usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + usleep(mInactivitySleepTime); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop. + mInactivitySleepTime *= 2; + + if(mInactivitySleepTime > MAX_INACTIVITY_SLEEP_TIME) + mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME; + return ; } + mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME; job = mFilesToHash.begin()->second ; @@ -145,3 +157,164 @@ void HashStorage::clean() #endif } +void HashStorage::load() +{ + uint64_t file_size ; + + if(!RsDirUtil::checkFile( mFilePath,file_size,false ) ) + { + std::cerr << "Encrypted hash cache file not present." << std::endl; + return ; + } + + // read the binary stream into memory. + // + void *buffer = rs_malloc(file_size) ; + + if(buffer == NULL) + return ; + + FILE *F = fopen( mFilePath.c_str(),"rb") ; + if (!F) + { + std::cerr << "Cannot open file for reading encrypted file cache, filename " << mFilePath << std::endl; + free(buffer); + return; + } + if(fread(buffer,1,file_size,F) != file_size) + { + std::cerr << "Cannot read from file " + mFilePath << ": something's wrong." << std::endl; + free(buffer) ; + fclose(F) ; + return ; + } + fclose(F) ; + + // now decrypt + void *decrypted_data =NULL; + int decrypted_data_size =0; + + if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size)) + { + std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl; + free(buffer) ; + return ; + } + free(buffer) ; + +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Loading HashCache from file " << mFilePath << std::endl ; + int n=0 ; +#endif + unsigned char *data = (unsigned char*)decrypted_data ; + uint32_t offset = 0 ; + HashStorageInfo info ; + + while(offset < decrypted_data_size) + if(readHashStorageInfo(data,decrypted_data_size,offset,info)) + { +#ifdef HASHSTORAGE_DEBUG + std::cerr << info << std::endl; + ++n ; +#endif + mFiles[info.filename] = info ; + } + + free(decrypted_data) ; +#ifdef HASHSTORAGE_DEBUG + std::cerr << n << " entries loaded." << std::endl ; +#endif +} + +void HashStorage::save() +{ +#ifdef HASHSTORAGE_DEBUG + std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ; +#endif + + unsigned char *data = NULL ; + uint32_t offset = 0 ; + uint32_t total_size = 0; + + for(std::map::const_iterator it(mFiles.begin());it!=mFiles.end();++it) + writeHashStorageInfo(it->second,data,total_size,offset) ; + + void *encryptedData = NULL ; + int encDataLen = 0 ; + + if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,offset, AuthSSL::getAuthSSL()->OwnId())) + { + std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl; + return; + } + + FILE *F = fopen( (mFilePath+".tmp").c_str(),"wb" ) ; + + if(!F) + { + std::cerr << "Cannot open encrypted file cache for writing: " << mFilePath+".tmp" << std::endl; + goto save_free; + } + if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen) + { + std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl; + fclose(F) ; + goto save_free; + } + + fclose(F) ; + + RsDirUtil::renameFile(mFilePath+".tmp",mFilePath) ; +#ifdef FIM_DEBUG + std::cerr << "done." << std::endl ; +#endif + +save_free: + free(encryptedData); +} + +bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const +{ + unsigned char *section_data = NULL ; + uint32_t section_size = 0; + + if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size)) + return false; + + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ; + if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ; + + free(section_data); + return true; +} + +bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const +{ + unsigned char *section_data = NULL ; + uint32_t section_offset = 0 ; + uint32_t section_size = 0; + + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ; + if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ; + + // now write the whole string into a single section in the file + + if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ; + + free(section_data) ; + + return true; +} + +std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& info) +{ + return o << info.hash << " " << info.size << " " << info.filename ; +} + + diff --git a/libretroshare/src/file_sharing/hash_cache.h b/libretroshare/src/file_sharing/hash_cache.h index 2d10c09ca..cf6b12a9f 100644 --- a/libretroshare/src/file_sharing/hash_cache.h +++ b/libretroshare/src/file_sharing/hash_cache.h @@ -34,7 +34,7 @@ public: struct HashStorageInfo { - std::string filename ; + std::string filename ; // full path of the file uint64_t size ; uint32_t time_stamp ; // last time the hash was tested/requested uint32_t modf_stamp ; @@ -50,12 +50,17 @@ public: // Functions called by the thread virtual void data_tick() ; + + friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ; private: void clean() ; void save() ; void load() ; + bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const; + bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const; + // Local configuration and storage uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries @@ -79,5 +84,6 @@ private: RsMutex mHashMtx ; bool mRunning; + uint32_t mInactivitySleepTime = 50*1000 ; }; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index b7a113fd7..70994b8c7 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -38,12 +38,14 @@ CONFIG += file_lists file_lists { HEADERS *= file_sharing/p3filelists.h \ file_sharing/hash_cache.h \ - file_sharing/directory_storage.h \ + file_sharing/filelist_io.h \ + file_sharing/directory_storage.h \ file_sharing/directory_updater.h SOURCES *= file_sharing/p3filelists.cc \ file_sharing/hash_cache.cc \ - file_sharing/directory_storage.cc \ + file_sharing/filelist_io.cc \ + file_sharing/directory_storage.cc \ file_sharing/directory_updater.cc }