From aff10a352cd701a3bede97f44d0c86222c96105a Mon Sep 17 00:00:00 2001 From: thunder2 Date: Sat, 4 Jan 2014 22:39:34 +0000 Subject: [PATCH] Optimized filling of shared files (local and remote). Added cache to RemoteDirModel. git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@6989 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/dbase/findex.cc | 14 +- libretroshare/src/dbase/fistore.cc | 6 +- retroshare-gui/src/gui/RemoteDirModel.cpp | 226 +++++++++++----------- retroshare-gui/src/gui/RemoteDirModel.h | 15 +- 4 files changed, 140 insertions(+), 121 deletions(-) diff --git a/libretroshare/src/dbase/findex.cc b/libretroshare/src/dbase/findex.cc index d04101213..81930c944 100644 --- a/libretroshare/src/dbase/findex.cc +++ b/libretroshare/src/dbase/findex.cc @@ -1340,7 +1340,7 @@ bool FileIndex::extractData(void *ref,DirDetails& details) FileEntry *file = static_cast(ref); DirEntry *dir = (file->hash.empty())?static_cast(file):NULL ; // This is a hack to avoid doing a dynamic_cast - details.children = std::list() ; + details.children.clear() ; time_t now = time(NULL) ; if (dir!=NULL) /* has children --- fill */ @@ -1351,20 +1351,24 @@ bool FileIndex::extractData(void *ref,DirDetails& details) /* extract all the entries */ for(std::map::const_iterator dit(dir->subdirs.begin()); dit != dir->subdirs.end(); ++dit) { + DirEntry *dirEntry = dit->second; + DirStub stub; stub.type = DIR_TYPE_DIR; - stub.name = (dit->second) -> name; - stub.ref = (dit->second); + stub.name = dirEntry -> name; + stub.ref = dirEntry; details.children.push_back(stub); } for(std::map::const_iterator fit(dir->files.begin()); fit != dir->files.end(); ++fit) { + FileEntry *fileEntry = fit->second; + DirStub stub; stub.type = DIR_TYPE_FILE; - stub.name = (fit->second) -> name; - stub.ref = (fit->second); + stub.name = fileEntry -> name; + stub.ref = fileEntry; details.children.push_back(stub); } diff --git a/libretroshare/src/dbase/fistore.cc b/libretroshare/src/dbase/fistore.cc index bbe06a7c8..78e35eb42 100644 --- a/libretroshare/src/dbase/fistore.cc +++ b/libretroshare/src/dbase/fistore.cc @@ -247,10 +247,12 @@ int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, FileSearch { /* */ + FileIndex *fileIndex = pit->second; + DirStub stub; stub.type = DIR_TYPE_PERSON; - stub.name = (pit->second)->root->name; - stub.ref = (pit->second)->root; + stub.name = fileIndex->root->name; + stub.ref = fileIndex->root; details.children.push_back(stub); } diff --git a/retroshare-gui/src/gui/RemoteDirModel.cpp b/retroshare-gui/src/gui/RemoteDirModel.cpp index 0d04b0c0a..af7be29ed 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.cpp +++ b/retroshare-gui/src/gui/RemoteDirModel.cpp @@ -100,10 +100,9 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const void *ref = parent.internalPointer(); - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) { /* error */ #ifdef RDM_DEBUG @@ -113,7 +112,7 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const return false; } - if (details.type == DIR_TYPE_FILE) + if (details->type == DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "lookup FILE -> false"; @@ -123,10 +122,10 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const } /* PERSON/DIR*/ #ifdef RDM_DEBUG - std::cerr << "lookup PER/DIR #" << details.count; + std::cerr << "lookup PER/DIR #" << details->count; std::cerr << std::endl; #endif - return (details.count > 0); /* do we have children? */ + return (details->count > 0); /* do we have children? */ } bool FlatStyle_RDM::hasChildren(const QModelIndex &parent) const { @@ -157,10 +156,9 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const void *ref = (parent.isValid())? parent.internalPointer() : NULL ; - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) { #ifdef RDM_DEBUG std::cerr << "lookup failed -> 0"; @@ -168,7 +166,7 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const #endif return 0; } - if (details.type == DIR_TYPE_FILE) + if (details->type == DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "lookup FILE: 0"; @@ -179,10 +177,10 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const /* else PERSON/DIR*/ #ifdef RDM_DEBUG - std::cerr << "lookup PER/DIR #" << details.count; + std::cerr << "lookup PER/DIR #" << details->count; std::cerr << std::endl; #endif - return details.count; + return details->count; } int FlatStyle_RDM::rowCount(const QModelIndex &parent) const @@ -384,23 +382,22 @@ QVariant TreeStyle_RDM::displayRole(const DirDetails& details,int coln) const QString FlatStyle_RDM::computeDirectoryPath(const DirDetails& details) const { QString dir ; - DirDetails det(details) ; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *det = requestDirDetails(details.parent,RemoteMode); - if(!requestDirDetails(det.parent,det,flags)) + if(!det) return QString(); #ifdef SHOW_TOTAL_PATH do { #endif - dir = QString::fromUtf8(det.name.c_str())+"/"+dir ; + dir = QString::fromUtf8(det->name.c_str())+"/"+dir ; #ifdef SHOW_TOTAL_PATH if(!requestDirDetails(det.parent,det,flags)) break ; } - while(det.parent != NULL); + while(det->parent != NULL); #endif return dir ; @@ -525,18 +522,17 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const void *ref = index.internalPointer(); int coln = index.column(); - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) return QVariant(); if (role == RetroshareDirModel::FileNameRole) /* end of FileNameRole */ - return QString::fromUtf8(details.name.c_str()); + return QString::fromUtf8(details->name.c_str()); if (role == Qt::TextColorRole) { - if(details.min_age > ageIndicator) + if(details->min_age > ageIndicator) return QVariant(QColor(Qt::gray)) ; else return QVariant() ; // standard @@ -544,7 +540,7 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const if(role == Qt::DecorationRole) - return decorationRole(details,coln) ; + return decorationRole(*details,coln) ; /***************** Qt::EditRole @@ -569,31 +565,29 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const } /* end of TextAlignmentRole */ if (role == Qt::DisplayRole) - return displayRole(details,coln) ; + return displayRole(*details,coln) ; if (role == SortRole) - return sortRole(index,details,coln) ; + return sortRole(index,*details,coln) ; return QVariant(); } -void RetroshareDirModel::getAgeIndicatorRec(DirDetails &details, QString &ret) const { - if (details.type == DIR_TYPE_FILE) { - ret = getAgeIndicatorString(details); - return; - } else if (details.type == DIR_TYPE_DIR && ret.isEmpty()) { - std::list::iterator it; - for (it = details.children.begin(); it != details.children.end(); it++) { - void *ref = it->ref; - DirDetails childDetails; +//void RetroshareDirModel::getAgeIndicatorRec(const DirDetails &details, QString &ret) const { +// if (details.type == DIR_TYPE_FILE) { +// ret = getAgeIndicatorString(details); +// return; +// } else if (details.type == DIR_TYPE_DIR && ret.isEmpty()) { +// std::list::const_iterator it; +// for (it = details.children.begin(); it != details.children.end(); it++) { +// void *ref = it->ref; +// const DirDetails *childDetails = requestDirDetails(ref, RemoteMode); - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; - - if (requestDirDetails(ref, childDetails, flags) && ret == tr("")) - getAgeIndicatorRec(childDetails, ret); - } - } -} +// if (childDetails && ret == tr("")) +// getAgeIndicatorRec(*childDetails, ret); +// } +// } +//} QVariant TreeStyle_RDM::headerData(int section, Qt::Orientation orientation, int role) const { @@ -703,10 +697,9 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent } ********/ - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetailsVector *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) { #ifdef RDM_DEBUG std::cerr << "lookup failed -> invalid"; @@ -720,11 +713,7 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent * get the reference number */ - std::list::iterator it; - int i = 0; - for(it = details.children.begin(); ((i < row) && (it != details.children.end())); ++it,++i) ; - - if (it == details.children.end()) + if (row >= (int) details->childrenVector.size()) { #ifdef RDM_DEBUG std::cerr << "wrong number of children -> invalid"; @@ -734,13 +723,13 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent } #ifdef RDM_DEBUG - std::cerr << "success index(" << row << "," << column << "," << it->ref << ")"; + std::cerr << "success index(" << row << "," << column << "," << details->childrenVector[row].ref << ")"; std::cerr << std::endl; #endif /* we can just grab the reference now */ - return createIndex(row, column, it->ref); + return createIndex(row, column, details->childrenVector[row].ref); } QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const { @@ -782,10 +771,9 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const } void *ref = index.internalPointer(); - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) { #ifdef RDM_DEBUG std::cerr << "Failed Lookup -> invalid"; @@ -794,7 +782,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const return QModelIndex(); } - if (!(details.parent)) + if (!(details->parent)) { #ifdef RDM_DEBUG std::cerr << "success. parent is Root/NULL --> invalid"; @@ -804,11 +792,11 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const } #ifdef RDM_DEBUG - std::cerr << "success index(" << details.prow << ",0," << details.parent << ")"; + std::cerr << "success index(" << details->prow << ",0," << details->parent << ")"; std::cerr << std::endl; #endif - return createIndex(details.prow, 0, details.parent); + return createIndex(details->prow, 0, details->parent); } QModelIndex FlatStyle_RDM::parent( const QModelIndex & index ) const { @@ -833,13 +821,12 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const void *ref = index.internalPointer(); - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) return Qt::ItemIsSelectable; // Error. - switch(details.type) + switch(details->type) { case DIR_TYPE_PERSON: return Qt::ItemIsEnabled; case DIR_TYPE_DIR: return Qt::ItemIsSelectable | Qt::ItemIsEnabled; @@ -869,6 +856,9 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const beginResetModel(); #endif layoutAboutToBeChanged(); + + /* Clear caches */ + mCache.clear(); } /* Callback from */ @@ -884,12 +874,31 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const #endif } -bool RetroshareDirModel::requestDirDetails(void *ref,DirDetails& details,FileSearchFlags flags) const +const DirDetailsVector *RetroshareDirModel::requestDirDetails(void *ref, bool remote) const { - // We should use a cache instead of calling RsFiles::RequestDirDetails(), which is very costly - // due to some pointer checking crap. + const QMap::const_iterator it = mCache.constFind(ref); + if (it != mCache.constEnd()) { + /* Details found in cache */ + return &it.value(); + } - return rsFiles->RequestDirDetails(ref, details, flags) ; + /* Get details from the lib */ + DirDetailsVector details; + FileSearchFlags flags = (remote) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL; + if (rsFiles->RequestDirDetails(ref, details, flags)) { + /* Convert std::list to std::vector for fast access with index */ + std::list::const_iterator childIt; + for (childIt = details.children.begin(); childIt != details.children.end(); ++childIt) { + details.childrenVector.push_back(*childIt); + } + + /* Add to cache, must cast to none const */ + const QMap::iterator it1 = ((QMap*) &mCache)->insert(ref, details); + return &it1.value(); + } + + /* No details found */ + return NULL; } void RetroshareDirModel::createCollectionFile(QWidget *parent, const QModelIndexList &list) @@ -977,12 +986,11 @@ void RetroshareDirModel::downloadDirectory(const DirDetails & dirDetails, int pr { if (!it->ref) continue; - DirDetails subDirDetails; - FileSearchFlags flags = RS_FILE_HINTS_REMOTE ; + const DirDetails *subDirDetails = requestDirDetails(it->ref, true); - if (!requestDirDetails(it->ref, subDirDetails, flags)) continue; + if (!subDirDetails) continue; - downloadDirectory(subDirDetails, prefixLen); + downloadDirectory(*subDirDetails, prefixLen); } } } @@ -999,13 +1007,12 @@ void RetroshareDirModel::getDirDetailsFromSelect (const QModelIndexList &list, s { void *ref = it -> internalPointer(); - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) continue; - dirVec.push_back(details); + dirVec.push_back(*details); } } } @@ -1037,29 +1044,28 @@ void RetroshareDirModel::getFileInfoFromIndexList(const QModelIndexList& list, s { void *ref = it -> internalPointer(); - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) continue; - if(details.type == DIR_TYPE_PERSON) + if(details->type == DIR_TYPE_PERSON) continue ; #ifdef RDM_DEBUG std::cerr << "::::::::::::FileRecommend:::: " << std::endl; - std::cerr << "Name: " << details.name << std::endl; - std::cerr << "Hash: " << details.hash << std::endl; - std::cerr << "Size: " << details.count << std::endl; - std::cerr << "Path: " << details.path << std::endl; + std::cerr << "Name: " << details->name << std::endl; + std::cerr << "Hash: " << details->hash << std::endl; + std::cerr << "Size: " << details->count << std::endl; + std::cerr << "Path: " << details->path << std::endl; #endif // Note: for directories, the returned hash, is the peer id, so if we collect // dirs, we need to be a bit more conservative for the - if(already_in.find(details.hash+details.name) == already_in.end()) + if(already_in.find(details->hash+details->name) == already_in.end()) { - file_details.push_back(details) ; - already_in.insert(details.hash+details.name) ; + file_details.push_back(*details) ; + already_in.insert(details->hash+details->name) ; } } #ifdef RDM_DEBUG @@ -1133,9 +1139,9 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list internalPointer(); - DirDetails details; + const DirDetails *details = requestDirDetails(ref, false); - if (!requestDirDetails(ref, details, RS_FILE_HINTS_LOCAL)) + if (!details) { #ifdef RDM_DEBUG std::cerr << "getFilePaths() Bad Request" << std::endl; @@ -1143,7 +1149,7 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::listtype != DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "getFilePaths() Not File" << std::endl; @@ -1153,14 +1159,14 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::listname << std::endl; + std::cerr << "Hash: " << details->hash << std::endl; + std::cerr << "Size: " << details->count << std::endl; + std::cerr << "Path: " << details->path << std::endl; #endif - std::string filepath = details.path + "/"; - filepath += details.name; + std::string filepath = details->path + "/"; + filepath += details->name; #ifdef RDM_DEBUG std::cerr << "Constructed FilePath: " << filepath << std::endl; @@ -1188,23 +1194,22 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con { void *ref = it -> internalPointer(); - DirDetails details; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode); - if (!requestDirDetails(ref, details, flags)) + if (!details) { continue; } #ifdef RDM_DEBUG std::cerr << "::::::::::::FileDrag:::: " << std::endl; - std::cerr << "Name: " << details.name << std::endl; - std::cerr << "Hash: " << details.hash << std::endl; - std::cerr << "Size: " << details.count << std::endl; - std::cerr << "Path: " << details.path << std::endl; + std::cerr << "Name: " << details->name << std::endl; + std::cerr << "Hash: " << details->hash << std::endl; + std::cerr << "Size: " << details->count << std::endl; + std::cerr << "Path: " << details->path << std::endl; #endif - if (details.type != DIR_TYPE_FILE) + if (details->type != DIR_TYPE_FILE) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::mimeData() Not File" << std::endl; @@ -1212,7 +1217,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con continue; /* not file! */ } - if (drags.end() != (dit = drags.find(details.hash))) + if (drags.end() != (dit = drags.find(details->hash))) { #ifdef RDM_DEBUG std::cerr << "RetroshareDirModel::mimeData() Duplicate" << std::endl; @@ -1220,9 +1225,9 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con continue; /* duplicate */ } - drags[details.hash] = details.count; + drags[details->hash] = details->count; - QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details.name.c_str()), QString::fromStdString(details.hash), QString::number(details.count)); + QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details->name.c_str()), QString::fromStdString(details->hash), QString::number(details->count)); if (RemoteMode) { @@ -1241,7 +1246,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con std::cerr << "Created MimeData:"; std::cerr << std::endl; - std::cerr << text; + std::cerr << text.toStdString(); std::cerr << std::endl; #endif @@ -1315,17 +1320,16 @@ void FlatStyle_RDM::updateRefs() std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl; #endif _ref_stack.pop_back() ; - DirDetails details ; - FileSearchFlags flags = (RemoteMode)?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL; + const DirDetails *details = requestDirDetails(ref, RemoteMode) ; - if (requestDirDetails(ref, details, flags)) + if (details) { - if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons. - _ref_entries.push_back(std::pair(ref,computeDirectoryPath(details))); + if(details->type == DIR_TYPE_FILE) // only push files, not directories nor persons. + _ref_entries.push_back(std::pair(ref,computeDirectoryPath(*details))); #ifdef RDM_DEBUG std::cerr << "FlatStyle_RDM::postMods(): addign ref " << ref << std::endl; #endif - for(std::list::iterator it = details.children.begin(); it != details.children.end(); it++) + for(std::list::const_iterator it = details->children.begin(); it != details->children.end(); it++) _ref_stack.push_back(it->ref) ; } if(++nb_treated_refs > MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to diff --git a/retroshare-gui/src/gui/RemoteDirModel.h b/retroshare-gui/src/gui/RemoteDirModel.h index bb76b2154..f1fb73a87 100644 --- a/retroshare-gui/src/gui/RemoteDirModel.h +++ b/retroshare-gui/src/gui/RemoteDirModel.h @@ -30,6 +30,14 @@ class DirDetails; +class DirDetailsVector : public DirDetails +{ +public: + DirDetailsVector() : DirDetails() {} + + std::vector childrenVector; // For fast access with index (can we change the std::list on DirDetails to std::vector?) +}; + static const uint32_t IND_LAST_DAY = 3600*24 ; static const uint32_t IND_LAST_WEEK = 3600*24*7 ; static const uint32_t IND_LAST_MONTH = 3600*24*31 ; // I know, this is approximate @@ -66,10 +74,10 @@ class RetroshareDirModel : public QAbstractItemModel void getFilePaths(const QModelIndexList &list, std::list &fullpaths); void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; } - bool requestDirDetails(void *ref,DirDetails& details,FileSearchFlags flags) const; + const DirDetailsVector *requestDirDetails(void *ref, bool remote) const; void update() ; - public: + public: virtual QMimeData * mimeData ( const QModelIndexList & indexes ) const; virtual QStringList mimeTypes () const; virtual QVariant data(const QModelIndex &index, int role) const; @@ -86,7 +94,7 @@ class RetroshareDirModel : public QAbstractItemModel static QString getFlagsString(FileStorageFlags f) ; static QString getGroupsString(const std::list&) ; QString getAgeIndicatorString(const DirDetails &) const; - void getAgeIndicatorRec(DirDetails &details, QString &ret) const; +// void getAgeIndicatorRec(const DirDetails &details, QString &ret) const; virtual QVariant displayRole(const DirDetails&,int) const = 0 ; virtual QVariant sortRole(const QModelIndex&,const DirDetails&,int) const =0; @@ -135,6 +143,7 @@ class RetroshareDirModel : public QAbstractItemModel }; bool RemoteMode; + QMap mCache; mutable int nIndex; mutable std::vector indexSet;