Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #1047 from csoler/v0.6-FT

Browse Source 
removed the 4M files limit on 64bits systems
This commit is contained in:
csoler 2017-09-21 23:47:11 +02:00 committed by GitHub
commit 73cbf6ca38
3 changed files with 87 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
libretroshare/src/file_sharing/file_sharing_defaults.h
View File

@ -53,9 +53,14 @@ static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; //
static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission
static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days
static const uint32_t NB_FRIEND_INDEX_BITS = 10 ; // Do not change this! static const uint32_t NB_FRIEND_INDEX_BITS_32BITS = 10 ; // Do not change this!
static const uint32_t NB_ENTRY_INDEX_BITS = 22 ; // Do not change this! static const uint32_t NB_ENTRY_INDEX_BITS_32BITS = 22 ; // Do not change this!
static const uint32_t ENTRY_INDEX_BIT_MASK = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. Depends on the two values just before. Dont change! static const uint32_t ENTRY_INDEX_BIT_MASK_32BITS = 0x003fffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. Depends on the two values just before. Dont change!
static const uint64_t NB_FRIEND_INDEX_BITS_64BITS = 24 ; // Do not change this!
static const uint64_t NB_ENTRY_INDEX_BITS_64BITS = 40 ; // Do not change this!
static const uint64_t ENTRY_INDEX_BIT_MASK_64BITS = 0x0000000fffffffff ; // used for storing (EntryIndex,Friend) couples into a 32bits pointer. Depends on the two values just before. Dont change!
static const uint32_t DELAY_BEFORE_DROP_REQUEST = 600; // every 10 min static const uint32_t DELAY_BEFORE_DROP_REQUEST = 600; // every 10 min
static const bool FOLLOW_SYMLINKS_DEFAULT = true; static const bool FOLLOW_SYMLINKS_DEFAULT = true;

101
libretroshare/src/file_sharing/p3filelists.cc
View File

@ -669,10 +669,15 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
#endif #endif
} }
if(mRemoteDirectories.size() >= (1 << NB_FRIEND_INDEX_BITS) ) if(sizeof(void*)==4 && mRemoteDirectories.size() >= (1u << NB_FRIEND_INDEX_BITS_32BITS) )
{ {
std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than " << (1<<NB_FRIEND_INDEX_BITS) << " friends??" << std::endl; std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than " << (1<<NB_FRIEND_INDEX_BITS_32BITS) << " friends??" << std::endl;
return 1 << NB_FRIEND_INDEX_BITS ; return 1 << NB_FRIEND_INDEX_BITS_32BITS ;
}
else if(sizeof(void*)==8 && mRemoteDirectories.size() >= (1ull << NB_FRIEND_INDEX_BITS_64BITS) )
{
std::cerr << "(EE) FriendIndexTab is full. This is weird. Do you really have more than " << (1<<NB_FRIEND_INDEX_BITS_64BITS) << " friends??" << std::endl;
return 1 << NB_FRIEND_INDEX_BITS_64BITS ;
} }
mFriendIndexMap[pid] = found; mFriendIndexMap[pid] = found;
@ -700,13 +705,13 @@ uint32_t p3FileDatabase::locked_getFriendIndex(const RsPeerId& pid)
} }
} }
bool p3FileDatabase::convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) template<> bool p3FileDatabase::convertPointerToEntryIndex<4>(const void *p, EntryIndex& e, uint32_t& friend_index)
{ {
// trust me, I can do this ;-) // trust me, I can do this ;-)
#pragma GCC diagnostic ignored "-Wstrict-aliasing" #pragma GCC diagnostic ignored "-Wstrict-aliasing"
e = EntryIndex( *reinterpret_cast<uint32_t*>(&p) & ENTRY_INDEX_BIT_MASK ) ; e = EntryIndex( *reinterpret_cast<uint32_t*>(&p) & ENTRY_INDEX_BIT_MASK_32BITS ) ;
friend_index = (*reinterpret_cast<uint32_t*>(&p)) >> NB_ENTRY_INDEX_BITS ; friend_index = (*reinterpret_cast<uint32_t*>(&p)) >> NB_ENTRY_INDEX_BITS_32BITS ;
#pragma GCC diagnostic pop #pragma GCC diagnostic pop
if(friend_index == 0) if(friend_index == 0)
@ -718,7 +723,7 @@ bool p3FileDatabase::convertPointerToEntryIndex(const void *p, EntryIndex& e, ui
return true; return true;
} }
bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi, void *& p) template<> bool p3FileDatabase::convertEntryIndexToPointer<4>(const EntryIndex& e, uint32_t fi, void *& p)
{ {
// the pointer is formed the following way: // the pointer is formed the following way:
// //
@ -730,23 +735,69 @@ bool p3FileDatabase::convertEntryIndexToPointer(const EntryIndex& e, uint32_t fi
uint32_t fe = (uint32_t)e ; uint32_t fe = (uint32_t)e ;
if(fi+1 >= (1<<NB_FRIEND_INDEX_BITS) || fe >= (1<< NB_ENTRY_INDEX_BITS)) if(fi+1 >= (1u<<NB_FRIEND_INDEX_BITS_32BITS) || fe >= (1u<< NB_ENTRY_INDEX_BITS_32BITS))
{ {
P3FILELISTS_ERROR() << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl; P3FILELISTS_ERROR() << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl;
return false ; return false ;
} }
p = reinterpret_cast<void*>( ( (1+fi) << NB_ENTRY_INDEX_BITS ) + (fe & ENTRY_INDEX_BIT_MASK)) ; p = reinterpret_cast<void*>( ( (1u+fi) << NB_ENTRY_INDEX_BITS_32BITS ) + (fe & ENTRY_INDEX_BIT_MASK_32BITS)) ;
return true; return true;
} }
template<> bool p3FileDatabase::convertPointerToEntryIndex<8>(const void *p, EntryIndex& e, uint32_t& friend_index)
{
// trust me, I can do this ;-)
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
e = EntryIndex( *reinterpret_cast<uint64_t*>(&p) & ENTRY_INDEX_BIT_MASK_64BITS ) ;
friend_index = (*reinterpret_cast<uint64_t*>(&p)) >> NB_ENTRY_INDEX_BITS_64BITS ;
#pragma GCC diagnostic pop
if(friend_index == 0)
{
std::cerr << "(EE) Cannot find friend index in pointer. Encoded value is zero!" << std::endl;
return false;
}
friend_index--;
#ifdef DEBUG_P3FILELISTS
std::cerr << "Generated index " << e <<" friend " << friend_index << " from pointer " << p << std::endl;
#endif
return true;
}
template<> bool p3FileDatabase::convertEntryIndexToPointer<8>(const EntryIndex& e, uint32_t fi, void *& p)
{
// the pointer is formed the following way:
//
// [ 24 bits | 40 bits ]
//
// This means that the whoel software has the following build-in limitation:
// * 1.6M friends
// * 1T shared files.
uint64_t fe = (uint64_t)e ;
if(fi+1 >= (1ull<<NB_FRIEND_INDEX_BITS_64BITS) || fe >= (1ull<< NB_ENTRY_INDEX_BITS_64BITS))
{
P3FILELISTS_ERROR() << "(EE) cannot convert entry index " << e << " of friend with index " << fi << " to pointer." << std::endl;
return false ;
}
p = reinterpret_cast<void*>( ( (1ull+fi) << NB_ENTRY_INDEX_BITS_64BITS ) + (fe & ENTRY_INDEX_BIT_MASK_64BITS)) ;
#ifdef DEBUG_P3FILELISTS
std::cerr << "Generated pointer " << p << " from friend " << fi << " and index " << e << std::endl;
#endif
return true;
}
void p3FileDatabase::requestDirUpdate(void *ref) void p3FileDatabase::requestDirUpdate(void *ref)
{ {
uint32_t fi; uint32_t fi;
DirectoryStorage::EntryIndex e ; DirectoryStorage::EntryIndex e ;
convertPointerToEntryIndex(ref,e,fi); convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
if(fi == 0) if(fi == 0)
return ; // not updating current directory (should we?) return ; // not updating current directory (should we?)
@ -775,13 +826,13 @@ bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result,
if(row != 0) if(row != 0)
return false ; return false ;
convertEntryIndexToPointer(0,0,result); convertEntryIndexToPointer<sizeof(void*)>(0,0,result);
return true ; return true ;
} }
else if((uint32_t)row < mRemoteDirectories.size()) else if((uint32_t)row < mRemoteDirectories.size())
{ {
convertEntryIndexToPointer(mRemoteDirectories[row]->root(),row+1,result); convertEntryIndexToPointer<sizeof(void*)>(mRemoteDirectories[row]->root(),row+1,result);
return true; return true;
} }
else else
@ -791,7 +842,7 @@ bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result,
uint32_t fi; uint32_t fi;
DirectoryStorage::EntryIndex e ; DirectoryStorage::EntryIndex e ;
convertPointerToEntryIndex(ref,e,fi); convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
// check consistency // check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL)))
@ -806,7 +857,7 @@ bool p3FileDatabase::findChildPointer( void *ref, int row, void *& result,
EntryIndex c = 0; EntryIndex c = 0;
bool res = storage->getChildIndex(e,row,c); bool res = storage->getChildIndex(e,row,c);
convertEntryIndexToPointer(c,fi,result) ; convertEntryIndexToPointer<sizeof(void*)>(c,fi,result) ;
return res; return res;
} }
@ -870,7 +921,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
if(flags & RS_FILE_HINTS_LOCAL) if(flags & RS_FILE_HINTS_LOCAL)
{ {
void *p; void *p;
convertEntryIndexToPointer(0,0,p); convertEntryIndexToPointer<sizeof(void*)>(0,0,p);
DirStub stub; DirStub stub;
stub.type = DIR_TYPE_PERSON; stub.type = DIR_TYPE_PERSON;
@ -882,7 +933,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
if(mRemoteDirectories[i] != NULL) if(mRemoteDirectories[i] != NULL)
{ {
void *p; void *p;
convertEntryIndexToPointer(mRemoteDirectories[i]->root(),i+1,p); convertEntryIndexToPointer<sizeof(void*)>(mRemoteDirectories[i]->root(),i+1,p);
DirStub stub; DirStub stub;
stub.type = DIR_TYPE_PERSON; stub.type = DIR_TYPE_PERSON;
@ -903,7 +954,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
uint32_t fi; uint32_t fi;
DirectoryStorage::EntryIndex e ; DirectoryStorage::EntryIndex e ;
convertPointerToEntryIndex(ref,e,fi); convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
// check consistency // check consistency
if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL))) if( (fi == 0 && !(flags & RS_FILE_HINTS_LOCAL)) || (fi > 0 && (flags & RS_FILE_HINTS_LOCAL)))
@ -926,10 +977,10 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
// update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert // update indexes. This is a bit hacky, but does the job. The cast to intptr_t is the proper way to convert
// a pointer into an int. // a pointer into an int.
convertEntryIndexToPointer((intptr_t)d.ref,fi,d.ref) ; convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.ref,fi,d.ref) ;
for(uint32_t i=0;i<d.children.size();++i) for(uint32_t i=0;i<d.children.size();++i)
convertEntryIndexToPointer((intptr_t)d.children[i].ref,fi,d.children[i].ref); convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.children[i].ref,fi,d.children[i].ref);
if(e == 0) // root if(e == 0) // root
{ {
@ -943,7 +994,7 @@ int p3FileDatabase::RequestDirDetails(void *ref, DirDetails& d, FileSearchFlags
else else
d.prow = storage->parentRow(e) ; d.prow = storage->parentRow(e) ;
convertEntryIndexToPointer((intptr_t)d.parent,fi,d.parent) ; convertEntryIndexToPointer<sizeof(void*)>((intptr_t)d.parent,fi,d.parent) ;
} }
d.id = storage->peerId(); d.id = storage->peerId();
@ -976,7 +1027,7 @@ uint32_t p3FileDatabase::getType(void *ref) const
if(ref == NULL) if(ref == NULL)
return DIR_TYPE_ROOT ; return DIR_TYPE_ROOT ;
convertPointerToEntryIndex(ref,e,fi); convertPointerToEntryIndex<sizeof(void*)>(ref,e,fi);
if(e == 0) if(e == 0)
return DIR_TYPE_PERSON ; return DIR_TYPE_PERSON ;
@ -1056,7 +1107,7 @@ int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::
for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it) for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
{ {
void *p=NULL; void *p=NULL;
convertEntryIndexToPointer(*it,0,p); convertEntryIndexToPointer<sizeof(void*)>(*it,0,p);
*it = (intptr_t)p ; *it = (intptr_t)p ;
} }
} }
@ -1080,7 +1131,7 @@ int p3FileDatabase::SearchKeywords(const std::list<std::string>& keywords, std::
for(std::list<EntryIndex>::iterator it(local_results.begin());it!=local_results.end();++it) for(std::list<EntryIndex>::iterator it(local_results.begin());it!=local_results.end();++it)
{ {
void *p=NULL; void *p=NULL;
convertEntryIndexToPointer(*it,i+1,p); convertEntryIndexToPointer<sizeof(void*)>(*it,i+1,p);
firesults.push_back((intptr_t)p) ; firesults.push_back((intptr_t)p) ;
} }
} }
@ -1114,7 +1165,7 @@ int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::lis
for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it) for(std::list<EntryIndex>::iterator it(firesults.begin());it!=firesults.end();++it)
{ {
void *p=NULL; void *p=NULL;
convertEntryIndexToPointer(*it,0,p); convertEntryIndexToPointer<sizeof(void*)>(*it,0,p);
*it = (intptr_t)p ; *it = (intptr_t)p ;
} }
} }
@ -1138,7 +1189,7 @@ int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::lis
for(std::list<EntryIndex>::iterator it(local_results.begin());it!=local_results.end();++it) for(std::list<EntryIndex>::iterator it(local_results.begin());it!=local_results.end();++it)
{ {
void *p=NULL; void *p=NULL;
convertEntryIndexToPointer(*it,i+1,p); convertEntryIndexToPointer<sizeof(void*)>(*it,i+1,p);
firesults.push_back((intptr_t)p) ; firesults.push_back((intptr_t)p) ;
} }

5
libretroshare/src/file_sharing/p3filelists.h
View File

@ -205,8 +205,9 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
// utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc // utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc
static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p); template<int BYTES> static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p);
static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ; template<int BYTES> static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ;
uint32_t locked_getFriendIndex(const RsPeerId& pid); uint32_t locked_getFriendIndex(const RsPeerId& pid);
void handleDirSyncRequest (RsFileListsSyncRequestItem *) ; void handleDirSyncRequest (RsFileListsSyncRequestItem *) ;