Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

added load/save for file hash cache

Browse Source
This commit is contained in:
MrAlice 2016-08-09 15:07:02 +02:00
parent 2d1824b01d
commit 82c18bbb86
6 changed files with 245 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
libretroshare/src/file_sharing/directory_storage.h
View File

@ -7,15 +7,6 @@
#include "retroshare/rsids.h"
#include "retroshare/rsfiles.h"
static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ;
static const uint8_t DIRECTORY_STORAGE_TAG_FILE_HASH = 0x01 ;
static const uint8_t DIRECTORY_STORAGE_TAG_FILE_NAME = 0x02 ;
static const uint8_t DIRECTORY_STORAGE_TAG_FILE_SIZE = 0x03 ;
static const uint8_t DIRECTORY_STORAGE_TAG_DIR_NAME = 0x04 ;
static const uint8_t DIRECTORY_STORAGE_TAG_MODIF_TS = 0x05 ;
static const uint8_t DIRECTORY_STORAGE_TAG_RECURS_MODIF_TS = 0x06 ;
#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
class InternalFileHierarchyStorage ;

2
libretroshare/src/file_sharing/filelist_io.cc Normal file
View File

@ -0,0 +1,2 @@
#include "filelist_io.h"

58
libretroshare/src/file_sharing/filelist_io.h Normal file
View File

@ -0,0 +1,58 @@
#pragma once
#include <stdint.h>
// This file implements load/save of various fields used for file lists and directory content.
// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers.
static const uint8_t DIRECTORY_STORAGE_VERSION = 0x01 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x01 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x02 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x03 ;
static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x04 ;
static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x05 ;
static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x06 ;
static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x07 ;
static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x08 ;
class FileListIO
{
public:
template<typename T>
static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val)
{
if(!checkSectionSize(buff,buff_size,offset,sizeof(T)))
return false;
if(!writeSectionHeader(buff,buff_size,offset,section_tag,sizeof(T)))
return false;
memcpy(&buff[offset],reinterpret_cast<const uint8_t*>(&val),sizeof(T)) ;
offset += sizeof(T) ;
return true;
}
private:
static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S)
{
if(offset + S > buff_size)
{
buff = (unsigned char *)realloc(buff,offset + S) ;
buff_size = offset + S ;
if(!buff)
return false ;
}
return true ;
}
static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint8_t section_tag,uint32_t S)
{
buff[offset++] = section_tag ;
if(!write125Size(buff,offset,buff_size,S)) return false ;
return true;
}
};

175
libretroshare/src/file_sharing/hash_cache.cc
View File

@ -1,13 +1,19 @@
#include "util/rsdir.h"
#include "rsserver/p3face.h"
#include "pqi/authssl.h"
#include "hash_cache.h"
#include "filelist_io.h"
#define HASHSTORAGE_DEBUG 1
static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000;
static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000;
HashStorage::HashStorage(const std::string& save_file_name)
: mFilePath(save_file_name), mHashMtx("Hash Storage mutex")
{
mRunning = false ;
load() ;
}
void HashStorage::data_tick()
@ -28,9 +34,15 @@ void HashStorage::data_tick()
RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ;
usleep(2*1000*1000); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop.
usleep(mInactivitySleepTime); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop.
mInactivitySleepTime *= 2;
if(mInactivitySleepTime > MAX_INACTIVITY_SLEEP_TIME)
mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME;
return ;
}
mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME;
job = mFilesToHash.begin()->second ;
@ -145,3 +157,164 @@ void HashStorage::clean()
#endif
}
void HashStorage::load()
{
uint64_t file_size ;
if(!RsDirUtil::checkFile( mFilePath,file_size,false ) )
{
std::cerr << "Encrypted hash cache file not present." << std::endl;
return ;
}
// read the binary stream into memory.
//
void *buffer = rs_malloc(file_size) ;
if(buffer == NULL)
return ;
FILE *F = fopen( mFilePath.c_str(),"rb") ;
if (!F)
{
std::cerr << "Cannot open file for reading encrypted file cache, filename " << mFilePath << std::endl;
free(buffer);
return;
}
if(fread(buffer,1,file_size,F) != file_size)
{
std::cerr << "Cannot read from file " + mFilePath << ": something's wrong." << std::endl;
free(buffer) ;
fclose(F) ;
return ;
}
fclose(F) ;
// now decrypt
void *decrypted_data =NULL;
int decrypted_data_size =0;
if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size))
{
std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl;
free(buffer) ;
return ;
}
free(buffer) ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Loading HashCache from file " << mFilePath << std::endl ;
int n=0 ;
#endif
unsigned char *data = (unsigned char*)decrypted_data ;
uint32_t offset = 0 ;
HashStorageInfo info ;
while(offset < decrypted_data_size)
if(readHashStorageInfo(data,decrypted_data_size,offset,info))
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << info << std::endl;
++n ;
#endif
mFiles[info.filename] = info ;
}
free(decrypted_data) ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << n << " entries loaded." << std::endl ;
#endif
}
void HashStorage::save()
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ;
#endif
unsigned char *data = NULL ;
uint32_t offset = 0 ;
uint32_t total_size = 0;
for(std::map<std::string,HashStorageInfo>::const_iterator it(mFiles.begin());it!=mFiles.end();++it)
writeHashStorageInfo(it->second,data,total_size,offset) ;
void *encryptedData = NULL ;
int encDataLen = 0 ;
if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,offset, AuthSSL::getAuthSSL()->OwnId()))
{
std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl;
return;
}
FILE *F = fopen( (mFilePath+".tmp").c_str(),"wb" ) ;
if(!F)
{
std::cerr << "Cannot open encrypted file cache for writing: " << mFilePath+".tmp" << std::endl;
goto save_free;
}
if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen)
{
std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl;
fclose(F) ;
goto save_free;
}
fclose(F) ;
RsDirUtil::renameFile(mFilePath+".tmp",mFilePath) ;
#ifdef FIM_DEBUG
std::cerr << "done." << std::endl ;
#endif
save_free:
free(encryptedData);
}
bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const
{
unsigned char *section_data = NULL ;
uint32_t section_size = 0;
if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size))
return false;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ;
free(section_data);
return true;
}
bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const
{
unsigned char *section_data = NULL ;
uint32_t section_offset = 0 ;
uint32_t section_size = 0;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ;
// now write the whole string into a single section in the file
if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ;
free(section_data) ;
return true;
}
std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& info)
{
return o << info.hash << " " << info.size << " " << info.filename ;
}

8
libretroshare/src/file_sharing/hash_cache.h
View File

@ -34,7 +34,7 @@ public:
struct HashStorageInfo
{
std::string filename ;
std::string filename ; // full path of the file
uint64_t size ;
uint32_t time_stamp ; // last time the hash was tested/requested
uint32_t modf_stamp ;
@ -50,12 +50,17 @@ public:
// Functions called by the thread
virtual void data_tick() ;
friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ;
private:
void clean() ;
void save() ;
void load() ;
bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const;
bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const;
// Local configuration and storage
uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries
@ -79,5 +84,6 @@ private:
RsMutex mHashMtx ;
bool mRunning;
uint32_t mInactivitySleepTime = 50*1000 ;
};

6
libretroshare/src/libretroshare.pro
View File

@ -38,12 +38,14 @@ CONFIG += file_lists
file_lists {
HEADERS *= file_sharing/p3filelists.h \
file_sharing/hash_cache.h \
file_sharing/directory_storage.h \
file_sharing/filelist_io.h \
file_sharing/directory_storage.h \
file_sharing/directory_updater.h
SOURCES *= file_sharing/p3filelists.cc \
file_sharing/hash_cache.cc \
file_sharing/directory_storage.cc \
file_sharing/filelist_io.cc \
file_sharing/directory_storage.cc \
file_sharing/directory_updater.cc
}