diff --git a/retroshare-gui/src/gui/common/RsCollection.cpp b/retroshare-gui/src/gui/common/RsCollection.cpp index e7e329604..21079c114 100644 --- a/retroshare-gui/src/gui/common/RsCollection.cpp +++ b/retroshare-gui/src/gui/common/RsCollection.cpp @@ -45,19 +45,25 @@ RsCollection::RsCollection(QObject *parent) RsCollection::RsCollection(const RsFileTree& ft) { mFileTree = std::unique_ptr(new RsFileTree(ft)); + + for(uint64_t i=0;inumFiles();++i) + mHashes.insert(std::make_pair(mFileTree->fileData(i).hash,i )); } RsCollection::RsCollection(const std::vector& file_infos,FileSearchFlags flags, QObject *parent) : QObject(parent), mFileTree(new RsFileTree) { - if(! ( (flags & RS_FILE_HINTS_LOCAL) || (flags & RS_FILE_HINTS_REMOTE))) - { - std::cerr << "(EE) Wrong flags passed to RsCollection constructor. Please fix the code!" << std::endl; - return ; - } + if(! ( (flags & RS_FILE_HINTS_LOCAL) || (flags & RS_FILE_HINTS_REMOTE))) + { + std::cerr << "(EE) Wrong flags passed to RsCollection constructor. Please fix the code!" << std::endl; + return ; + } - for(uint32_t i = 0;iroot(),file_infos[i],flags) ; + + for(uint64_t i=0;inumFiles();++i) + mHashes.insert(std::make_pair(mFileTree->fileData(i).hash,i )); } RsCollection::~RsCollection() @@ -67,88 +73,88 @@ RsCollection::~RsCollection() void RsCollection::downloadFiles() const { #ifdef TODO_COLLECTION - // print out the element names of all elements that are direct children - // of the outermost element. - QDomElement docElem = _xml_doc.documentElement(); + // print out the element names of all elements that are direct children + // of the outermost element. + QDomElement docElem = _xml_doc.documentElement(); - std::vector colFileInfos ; + std::vector colFileInfos ; - recursCollectColFileInfos(docElem,colFileInfos,QString(),false) ; + recursCollectColFileInfos(docElem,colFileInfos,QString(),false) ; - RsCollectionDialog(_fileName, colFileInfos, false).exec() ; + RsCollectionDialog(_fileName, colFileInfos, false).exec() ; #endif } void RsCollection::autoDownloadFiles() const { #ifdef TODO_COLLECTION - QDomElement docElem = _xml_doc.documentElement(); + QDomElement docElem = _xml_doc.documentElement(); - std::vector colFileInfos; + std::vector colFileInfos; - recursCollectColFileInfos(docElem,colFileInfos,QString(),false); + recursCollectColFileInfos(docElem,colFileInfos,QString(),false); - QString dlDir = QString::fromUtf8(rsFiles->getDownloadDirectory().c_str()); + QString dlDir = QString::fromUtf8(rsFiles->getDownloadDirectory().c_str()); - foreach(ColFileInfo colFileInfo, colFileInfos) - { - autoDownloadFiles(colFileInfo, dlDir); - } + foreach(ColFileInfo colFileInfo, colFileInfos) + { + autoDownloadFiles(colFileInfo, dlDir); + } #endif } void RsCollection::autoDownloadFiles(ColFileInfo colFileInfo, QString dlDir) const { - if (!colFileInfo.filename_has_wrong_characters) - { - QString cleanPath = dlDir + colFileInfo.path ; - std::cout << "making directory " << cleanPath.toStdString() << std::endl; + if (!colFileInfo.filename_has_wrong_characters) + { + QString cleanPath = dlDir + colFileInfo.path ; + std::cout << "making directory " << cleanPath.toStdString() << std::endl; - if(!QDir(QApplication::applicationDirPath()).mkpath(cleanPath)) - std::cerr << "Unable to make path: " + cleanPath.toStdString() << std::endl; + if(!QDir(QApplication::applicationDirPath()).mkpath(cleanPath)) + std::cerr << "Unable to make path: " + cleanPath.toStdString() << std::endl; - if (colFileInfo.type==DIR_TYPE_FILE) - rsFiles->FileRequest(colFileInfo.name.toUtf8().constData(), - RsFileHash(colFileInfo.hash.toStdString()), - colFileInfo.size, - cleanPath.toUtf8().constData(), - RS_FILE_REQ_ANONYMOUS_ROUTING, - std::list()); - } - foreach(ColFileInfo colFileInfoChild, colFileInfo.children) - { - autoDownloadFiles(colFileInfoChild, dlDir); - } + if (colFileInfo.type==DIR_TYPE_FILE) + rsFiles->FileRequest(colFileInfo.name.toUtf8().constData(), + RsFileHash(colFileInfo.hash.toStdString()), + colFileInfo.size, + cleanPath.toUtf8().constData(), + RS_FILE_REQ_ANONYMOUS_ROUTING, + std::list()); + } + foreach(ColFileInfo colFileInfoChild, colFileInfo.children) + { + autoDownloadFiles(colFileInfoChild, dlDir); + } } static QString purifyFileName(const QString& input,bool& bad) { - static const QString bad_chars = "/\\\"*:?<>|" ; - bad = false ; - QString output = input ; + static const QString bad_chars = "/\\\"*:?<>|" ; + bad = false ; + QString output = input ; - for(int i=0;iaddFile(mFileTree->root(),fname.toStdString(),hash,size); #ifdef TO_REMOVE - ColFileInfo info ; - info.type = DIR_TYPE_FILE ; - info.name = fname ; - info.size = size ; - info.hash = QString::fromStdString(hash.toStdString()) ; + ColFileInfo info ; + info.type = DIR_TYPE_FILE ; + info.name = fname ; + info.size = size ; + info.hash = QString::fromStdString(hash.toStdString()) ; - recursAddElements(_xml_doc,info,_root) ; + recursAddElements(_xml_doc,info,_root) ; #endif } void RsCollection::merge_in(const RsFileTree& tree) @@ -175,56 +181,56 @@ void RsCollection::recursMergeTree(RsFileTree::DirIndex parent,const RsFileTree& #ifdef TO_REMOVE void RsCollection::recursCollectColFileInfos(const QDomElement& e,std::vector& colFileInfos,const QString& current_path, bool bad_chars_in_parent) const { - QDomNode n = e.firstChild() ; + QDomNode n = e.firstChild() ; #ifdef COLLECTION_DEBUG - std::cerr << "Parsing element " << e.tagName().toStdString() << std::endl; + std::cerr << "Parsing element " << e.tagName().toStdString() << std::endl; #endif - while(!n.isNull()) - { - QDomElement ee = n.toElement(); // try to convert the node to an element. + while(!n.isNull()) + { + QDomElement ee = n.toElement(); // try to convert the node to an element. #ifdef COLLECTION_DEBUG - std::cerr << " Seeing child " << ee.tagName().toStdString() << std::endl; + std::cerr << " Seeing child " << ee.tagName().toStdString() << std::endl; #endif - if(ee.tagName() == QString("File")) - { - ColFileInfo newChild ; - newChild.hash = ee.attribute(QString("sha1")) ; - bool bad_chars_detected = false ; - newChild.name = purifyFileName(ee.attribute(QString("name")), bad_chars_detected) ; - newChild.filename_has_wrong_characters = bad_chars_detected || bad_chars_in_parent ; - newChild.size = ee.attribute(QString("size")).toULongLong() ; - newChild.path = current_path ; - newChild.type = DIR_TYPE_FILE ; + if(ee.tagName() == QString("File")) + { + ColFileInfo newChild ; + newChild.hash = ee.attribute(QString("sha1")) ; + bool bad_chars_detected = false ; + newChild.name = purifyFileName(ee.attribute(QString("name")), bad_chars_detected) ; + newChild.filename_has_wrong_characters = bad_chars_detected || bad_chars_in_parent ; + newChild.size = ee.attribute(QString("size")).toULongLong() ; + newChild.path = current_path ; + newChild.type = DIR_TYPE_FILE ; - colFileInfos.push_back(newChild) ; - } - else if(ee.tagName() == QString("Directory")) - { - ColFileInfo newParent ; - bool bad_chars_detected = false ; - QString cleanDirName = purifyFileName(ee.attribute(QString("name")),bad_chars_detected) ; - newParent.name=cleanDirName; - newParent.filename_has_wrong_characters = bad_chars_detected || bad_chars_in_parent ; - newParent.size = 0; - newParent.path = current_path ; - newParent.type = DIR_TYPE_DIR ; + colFileInfos.push_back(newChild) ; + } + else if(ee.tagName() == QString("Directory")) + { + ColFileInfo newParent ; + bool bad_chars_detected = false ; + QString cleanDirName = purifyFileName(ee.attribute(QString("name")),bad_chars_detected) ; + newParent.name=cleanDirName; + newParent.filename_has_wrong_characters = bad_chars_detected || bad_chars_in_parent ; + newParent.size = 0; + newParent.path = current_path ; + newParent.type = DIR_TYPE_DIR ; - recursCollectColFileInfos(ee,newParent.children,current_path + "/" + cleanDirName, bad_chars_in_parent || bad_chars_detected) ; - uint32_t size = newParent.children.size(); - for(uint32_t i=0;iaddFile(parent,dd.name,dd.hash,dd.size); else if (dd.type == DIR_TYPE_DIR) - { + { RsFileTree::DirIndex new_dir_index = mFileTree->addDirectory(parent,dd.name); for(uint32_t i=0;iRequestDirDetails(dd.children[i].ref, subDirDetails, flags)) - continue; + continue; recursAddElements(new_dir_index,subDirDetails,flags) ; - } - } + } + } } #ifdef TO_REMOVE void RsCollection::recursAddElements(QDomDocument& doc,const ColFileInfo& colFileInfo,QDomElement& e) const { - if (colFileInfo.type == DIR_TYPE_FILE) - { - QDomElement f = doc.createElement("File") ; + if (colFileInfo.type == DIR_TYPE_FILE) + { + QDomElement f = doc.createElement("File") ; - f.setAttribute(QString("name"),colFileInfo.name) ; - f.setAttribute(QString("sha1"),colFileInfo.hash) ; - f.setAttribute(QString("size"),QString::number(colFileInfo.size)) ; + f.setAttribute(QString("name"),colFileInfo.name) ; + f.setAttribute(QString("sha1"),colFileInfo.hash) ; + f.setAttribute(QString("size"),QString::number(colFileInfo.size)) ; - e.appendChild(f) ; - } - else if (colFileInfo.type == DIR_TYPE_DIR) - { - QDomElement d = doc.createElement("Directory") ; + e.appendChild(f) ; + } + else if (colFileInfo.type == DIR_TYPE_DIR) + { + QDomElement d = doc.createElement("Directory") ; - d.setAttribute(QString("name"),colFileInfo.name) ; + d.setAttribute(QString("name"),colFileInfo.name) ; - for (std::vector::const_iterator it = colFileInfo.children.begin(); it != colFileInfo.children.end(); ++it) - recursAddElements(doc,(*it),d) ; + for (std::vector::const_iterator it = colFileInfo.children.begin(); it != colFileInfo.children.end(); ++it) + recursAddElements(doc,(*it),d) ; - e.appendChild(d) ; - } + e.appendChild(d) ; + } } void RsCollection::recursAddElements( QDomDocument& doc, const RsFileTree& ft, uint32_t index, QDomElement& e ) const { - std::vector subdirs; - std::vector subfiles ; - std::string name; - if(!ft.getDirectoryContent(name, subdirs, subfiles, index)) return; + std::vector subdirs; + std::vector subfiles ; + std::string name; + if(!ft.getDirectoryContent(name, subdirs, subfiles, index)) return; - QDomElement d = doc.createElement("Directory") ; - d.setAttribute(QString("name"),QString::fromUtf8(name.c_str())) ; - e.appendChild(d) ; + QDomElement d = doc.createElement("Directory") ; + d.setAttribute(QString("name"),QString::fromUtf8(name.c_str())) ; + e.appendChild(d) ; - for (uint32_t i=0;inumFiles();++i) + mHashes.insert(std::make_pair(mFileTree->fileData(i).hash,i )); + return true; } bool RsCollection::recursExportToXml(QDomDocument& doc,QDomElement& e,const RsFileTree::DirData& dd) const @@ -581,16 +591,16 @@ bool RsCollection::recursExportToXml(QDomDocument& doc,QDomElement& e,const RsFi #ifdef TO_REMOVE bool RsCollection::save(QWidget *parent) const { - QString fileName; - if(!misc::getSaveFileName(parent, RshareSettings::LASTDIR_EXTRAFILE, QApplication::translate("RsCollectionFile", "Create collection file"), QApplication::translate("RsCollectionFile", "Collection files") + " (*." + RsCollection::ExtensionString + ")", fileName)) - return false; + QString fileName; + if(!misc::getSaveFileName(parent, RshareSettings::LASTDIR_EXTRAFILE, QApplication::translate("RsCollectionFile", "Create collection file"), QApplication::translate("RsCollectionFile", "Collection files") + " (*." + RsCollection::ExtensionString + ")", fileName)) + return false; - if (!fileName.endsWith("." + RsCollection::ExtensionString)) - fileName += "." + RsCollection::ExtensionString ; + if (!fileName.endsWith("." + RsCollection::ExtensionString)) + fileName += "." + RsCollection::ExtensionString ; - std::cerr << "Got file name: " << fileName.toStdString() << std::endl; + std::cerr << "Got file name: " << fileName.toStdString() << std::endl; - return save(fileName); + return save(fileName); } @@ -606,38 +616,59 @@ qulonglong RsCollection::size() return mFileTree->totalFileSize(); #ifdef TO_REMOVE - QDomElement docElem = _xml_doc.documentElement(); + QDomElement docElem = _xml_doc.documentElement(); - std::vector colFileInfos; - recursCollectColFileInfos(docElem, colFileInfos, QString(),false); + std::vector colFileInfos; + recursCollectColFileInfos(docElem, colFileInfos, QString(),false); - uint64_t size = 0; + uint64_t size = 0; - for (uint32_t i = 0; i < colFileInfos.size(); ++i) { - size += colFileInfos[i].size; - } + for (uint32_t i = 0; i < colFileInfos.size(); ++i) { + size += colFileInfos[i].size; + } - return size; + return size; #endif } bool RsCollection::isCollectionFile(const QString &fileName) { - QString ext = QFileInfo(fileName).suffix().toLower(); + QString ext = QFileInfo(fileName).suffix().toLower(); - return (ext == RsCollection::ExtensionString); + return (ext == RsCollection::ExtensionString); } +void RsCollection::updateHashes(const std::map& old_to_new_hashes) +{ + for(auto it:old_to_new_hashes) + { + auto fit = mHashes.find(it.first); + + if(fit == mHashes.end()) + { + RsErr() << "Could not find hash " << it.first << " in RsCollection list of hashes. This is a bug." ; + return ; + } + const auto& fd(mFileTree->fileData(fit->second)); + + if(fd.hash != it.first) + { + RsErr() << "Mismatch hash for file " << fd.name << " (found " << fd.hash << " instead of " << it.first << ") in RsCollection list of hashes. This is a bug." ; + return ; + } + mFileTree->updateFile(fit->second,fd.name,it.second,fd.size); + } +} #ifdef TO_REMOVE void RsCollection::saveColl(std::vector colFileInfos, const QString &fileName) { - QDomElement root = _xml_doc.elementsByTagName("RsCollection").at(0).toElement(); - while (root.childNodes().count()>0) root.removeChild(root.firstChild()); - for(uint32_t i = 0;i0) root.removeChild(root.firstChild()); + for(uint32_t i = 0;i& old_to_new_hashes); private: bool recursExportToXml(QDomDocument& doc,QDomElement& e,const RsFileTree::DirData& dd) const; @@ -130,6 +131,8 @@ private: void autoDownloadFiles(ColFileInfo colFileInfo, QString dlDir) const ; std::unique_ptr mFileTree; + std::map mHashes; // used to efficiently update files being hashed + #ifdef TO_REMOVE QDomDocument _xml_doc ; QString _fileName ; diff --git a/retroshare-gui/src/gui/common/RsCollectionDialog.cpp b/retroshare-gui/src/gui/common/RsCollectionDialog.cpp index 71e1c2c75..d6ec70f22 100644 --- a/retroshare-gui/src/gui/common/RsCollectionDialog.cpp +++ b/retroshare-gui/src/gui/common/RsCollectionDialog.cpp @@ -23,6 +23,7 @@ #include "RsCollection.h" #include "util/misc.h" +#include "util/rsdir.h" #include #include @@ -738,7 +739,7 @@ void RsCollectionDialog::addSelectionRecursive() addSelection(true); } -static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree::DirIndex dir_index,bool recursive) +static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree::DirIndex dir_index,bool recursive,QSet& paths_to_hash) { QFileInfo fileInfo = path; @@ -746,17 +747,28 @@ static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree: { auto di = tree.addDirectory(dir_index,fileInfo.fileName().toUtf8().constData()); - QDir dirParent = fileInfo.absoluteFilePath(); - dirParent.setFilter(QDir::AllEntries | QDir::NoSymLinks | QDir::NoDotAndDotDot); - QFileInfoList childrenList = dirParent.entryInfoList(); + if(recursive) + { + QDir dirParent = fileInfo.absoluteFilePath(); + dirParent.setFilter(QDir::AllEntries | QDir::NoSymLinks | QDir::NoDotAndDotDot); + QFileInfoList childrenList = dirParent.entryInfoList(); - for(QFileInfo f:childrenList) - recursBuildFileTree(f.absoluteFilePath(),tree,di,recursive); + for(QFileInfo f:childrenList) + recursBuildFileTree(f.absoluteFilePath(),tree,di,recursive,paths_to_hash); + } } else { -#warning TODO: compute the hash - tree.addFile(dir_index,fileInfo.fileName().toUtf8().constData(),RsFileHash(),fileInfo.size()); + // Here we use a temporary hash that serves two purposes: + // 1 - identify the file in the RsFileTree of the collection so that we can update its hash when calculated + // 2 - mark the file as being processed + // The hash s is computed to be the hash of the path of the file. The collection must take care of multiple instances. + + Sha1CheckSum s = RsDirUtil::sha1sum((uint8_t*)(fileInfo.filePath().toUtf8().constData()),fileInfo.filePath().toUtf8().size()); + + tree.addFile(dir_index,fileInfo.fileName().toUtf8().constData(),s,fileInfo.size()); + + paths_to_hash.insert(fileInfo.filePath().toUtf8().constData()); } } /** @@ -768,7 +780,6 @@ static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree: */ void RsCollectionDialog::addSelection(bool recursive) { - QStringList fileToHash; QMap dirToAdd; int count=0;//to not scan all items on list .count() @@ -776,11 +787,13 @@ void RsCollectionDialog::addSelection(bool recursive) mCollectionModel->preMods(); + QSet paths_to_hash; // sha1sum of the paths to hash + foreach (QModelIndex index, milSelectionList) if(index.column()==0) //Get only FileName { RsFileTree tree; - recursBuildFileTree(_dirModel->filePath(_tree_proxyModel->mapToSource(index)),tree,tree.root(),recursive); + recursBuildFileTree(_dirModel->filePath(_tree_proxyModel->mapToSource(index)),tree,tree.root(),recursive,paths_to_hash); mCollection->merge_in(tree); @@ -852,9 +865,18 @@ void RsCollectionDialog::addSelection(bool recursive) // Process Files once all done ui._hashBox->addAttachments(fileToHash,RS_FILE_REQ_ANONYMOUS_ROUTING /*, 0*/); #endif +// std::map paths_and_hashes; +// for(auto path:paths_to_hash) +// paths_and_hashes.insert(std::make_pair(RsDirUtil::sha1sum((uint8_t*)path.toUtf8().constData(),path.toUtf8().size()),path)); + +// mCollectionModel->addFilesToHash(paths_and_hashes); + mCollectionModel->postMods(); + + ui._hashBox->addAttachments(QStringList(paths_to_hash.begin(),paths_to_hash.end()),RS_FILE_REQ_ANONYMOUS_ROUTING /*, 0*/); } +#ifdef TO_REMOVE /** * @brief RsCollectionDialog::addAllChild: Add children to RsCollection * @param fileInfoParent: Parent's QFileInfo to scan @@ -910,6 +932,7 @@ bool RsCollectionDialog::addAllChild(QFileInfo &fileInfoParent } return true; } +#endif /** * @brief RsCollectionDialog::remove: Remove selected Items in RSCollection @@ -1186,6 +1209,7 @@ void RsCollectionDialog::makeDir() */ void RsCollectionDialog::fileHashingFinished(QList hashedFiles) { +#ifdef TO_REMOVE std::cerr << "RsCollectionDialog::fileHashingFinished() started." << std::endl; QString message; @@ -1211,8 +1235,26 @@ void RsCollectionDialog::fileHashingFinished(QList hashedFiles) _newColFileInfos.push_back(colFileInfo); #endif } +#endif + // build a map of old-hash to new-hash for the hashed files, so that it can be passed to the mCollection for update - std::cerr << "RsCollectionDialog::fileHashingFinished message : " << message.toStdString() << std::endl; + std::map old_to_new_hashes; + + for(auto f:hashedFiles) + { + auto it = mFilesBeingHashed.find(f.filepath); + + if(it == mFilesBeingHashed.end()) + { + RsErr() << "Could not find hash-ID correspondence for path " << f.filepath.toUtf8().constData() << ". This is a bug." << std::endl; + continue; + } + old_to_new_hashes.insert(std::make_pair(it->second,f.hash)); + mFilesBeingHashed.erase(it); + } + mCollectionModel->preMods(); + mCollection->updateHashes(old_to_new_hashes); + mCollectionModel->postMods(); updateList(); } diff --git a/retroshare-gui/src/gui/common/RsCollectionDialog.h b/retroshare-gui/src/gui/common/RsCollectionDialog.h index 71c261cb4..9ba3ac857 100644 --- a/retroshare-gui/src/gui/common/RsCollectionDialog.h +++ b/retroshare-gui/src/gui/common/RsCollectionDialog.h @@ -18,6 +18,7 @@ * * *******************************************************************************/ +#include #include "ui_RsCollectionDialog.h" #include "RsCollection.h" #include "RsCollectionModel.h" @@ -90,12 +91,12 @@ private: bool addChild(QTreeWidgetItem *parent, const std::vector &child); bool removeItem(QTreeWidgetItem *item, bool &removeOnlyFile) ; void saveChild(QTreeWidgetItem *parentItem, ColFileInfo *parentInfo = NULL); -#endif - void addSelection(bool recursive) ; bool addAllChild(QFileInfo &fileInfoParent , QMap &dirToAdd , QStringList &fileToHash , int &count); +#endif + void addSelection(bool recursive) ; Ui::RsCollectionDialog ui; QString _fileName ; @@ -110,4 +111,6 @@ private: RsCollectionModel *mCollectionModel; RsCollection *mCollection; + + std::map mFilesBeingHashed; // map of file path vs. temporary ID used for the file while hashing }; diff --git a/retroshare-gui/src/gui/common/RsCollectionModel.cpp b/retroshare-gui/src/gui/common/RsCollectionModel.cpp index f05e25231..6e8352339 100644 --- a/retroshare-gui/src/gui/common/RsCollectionModel.cpp +++ b/retroshare-gui/src/gui/common/RsCollectionModel.cpp @@ -252,9 +252,9 @@ bool RsCollectionModel::setData(const QModelIndex& index,const QVariant& value,i if(role==Qt::CheckStateRole && convertInternalIdToIndex(index.internalId(), e)) { -//#ifdef DEBUG_COLLECTION_MODEL +#ifdef DEBUG_COLLECTION_MODEL std::cerr << "Setting check state of item " << index << " to " << value.toBool() << std::endl; -//#endif +#endif RsFileTree::DirIndex dir_index ; if(e.is_file) @@ -370,7 +370,6 @@ QVariant RsCollectionModel::checkStateRole(const EntryIndex& i,int col) const { if(i.is_file) { - std::cerr<< "entry is file, checkstate = " << (int)mFileInfos[i.index].is_checked << std::endl; if(mFileInfos[i.index].is_checked) return QVariant(Qt::Checked); else @@ -378,8 +377,6 @@ QVariant RsCollectionModel::checkStateRole(const EntryIndex& i,int col) const } else { - std::cerr<< "entry is dir, checkstate = " << (int)mDirInfos[i.index].check_state << std::endl; - switch(mDirInfos[i.index].check_state) { case SELECTED: return QVariant::fromValue((int)Qt::Checked);