added auto-update of files being hashed

This commit is contained in:
csoler 2024-03-12 21:39:05 +01:00
parent 5e3434396c
commit 7ef81a37ff
5 changed files with 266 additions and 190 deletions

View File

@ -45,6 +45,9 @@ RsCollection::RsCollection(QObject *parent)
RsCollection::RsCollection(const RsFileTree& ft)
{
mFileTree = std::unique_ptr<RsFileTree>(new RsFileTree(ft));
for(uint64_t i=0;i<mFileTree->numFiles();++i)
mHashes.insert(std::make_pair(mFileTree->fileData(i).hash,i ));
}
RsCollection::RsCollection(const std::vector<DirDetails>& file_infos,FileSearchFlags flags, QObject *parent)
@ -58,6 +61,9 @@ RsCollection::RsCollection(const std::vector<DirDetails>& file_infos,FileSearchF
for(uint32_t i = 0;i<file_infos.size();++i)
recursAddElements(mFileTree->root(),file_infos[i],flags) ;
for(uint64_t i=0;i<mFileTree->numFiles();++i)
mHashes.insert(std::make_pair(mFileTree->fileData(i).hash,i ));
}
RsCollection::~RsCollection()
@ -545,6 +551,10 @@ bool RsCollection::recursParseXml(QDomDocument& doc,const QDomNode& e,const RsFi
n = n.nextSibling() ;
}
mHashes.clear();
for(uint64_t i=0;i<mFileTree->numFiles();++i)
mHashes.insert(std::make_pair(mFileTree->fileData(i).hash,i ));
return true;
}
bool RsCollection::recursExportToXml(QDomDocument& doc,QDomElement& e,const RsFileTree::DirData& dd) const
@ -628,6 +638,27 @@ bool RsCollection::isCollectionFile(const QString &fileName)
return (ext == RsCollection::ExtensionString);
}
void RsCollection::updateHashes(const std::map<RsFileHash,RsFileHash>& old_to_new_hashes)
{
for(auto it:old_to_new_hashes)
{
auto fit = mHashes.find(it.first);
if(fit == mHashes.end())
{
RsErr() << "Could not find hash " << it.first << " in RsCollection list of hashes. This is a bug." ;
return ;
}
const auto& fd(mFileTree->fileData(fit->second));
if(fd.hash != it.first)
{
RsErr() << "Mismatch hash for file " << fd.name << " (found " << fd.hash << " instead of " << it.first << ") in RsCollection list of hashes. This is a bug." ;
return ;
}
mFileTree->updateFile(fit->second,fd.name,it.second,fd.size);
}
}
#ifdef TO_REMOVE
void RsCollection::saveColl(std::vector<ColFileInfo> colFileInfos, const QString &fileName)
{

View File

@ -103,6 +103,7 @@ public:
static bool isCollectionFile(const QString& fileName);
void updateHashes(const std::map<RsFileHash,RsFileHash>& old_to_new_hashes);
private:
bool recursExportToXml(QDomDocument& doc,QDomElement& e,const RsFileTree::DirData& dd) const;
@ -130,6 +131,8 @@ private:
void autoDownloadFiles(ColFileInfo colFileInfo, QString dlDir) const ;
std::unique_ptr<RsFileTree> mFileTree;
std::map<RsFileHash,RsFileTree::FileIndex> mHashes; // used to efficiently update files being hashed
#ifdef TO_REMOVE
QDomDocument _xml_doc ;
QString _fileName ;

View File

@ -23,6 +23,7 @@
#include "RsCollection.h"
#include "util/misc.h"
#include "util/rsdir.h"
#include <QCheckBox>
#include <QDateTime>
@ -738,7 +739,7 @@ void RsCollectionDialog::addSelectionRecursive()
addSelection(true);
}
static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree::DirIndex dir_index,bool recursive)
static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree::DirIndex dir_index,bool recursive,QSet<QString>& paths_to_hash)
{
QFileInfo fileInfo = path;
@ -746,17 +747,28 @@ static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree:
{
auto di = tree.addDirectory(dir_index,fileInfo.fileName().toUtf8().constData());
if(recursive)
{
QDir dirParent = fileInfo.absoluteFilePath();
dirParent.setFilter(QDir::AllEntries | QDir::NoSymLinks | QDir::NoDotAndDotDot);
QFileInfoList childrenList = dirParent.entryInfoList();
for(QFileInfo f:childrenList)
recursBuildFileTree(f.absoluteFilePath(),tree,di,recursive);
recursBuildFileTree(f.absoluteFilePath(),tree,di,recursive,paths_to_hash);
}
}
else
{
#warning TODO: compute the hash
tree.addFile(dir_index,fileInfo.fileName().toUtf8().constData(),RsFileHash(),fileInfo.size());
// Here we use a temporary hash that serves two purposes:
// 1 - identify the file in the RsFileTree of the collection so that we can update its hash when calculated
// 2 - mark the file as being processed
// The hash s is computed to be the hash of the path of the file. The collection must take care of multiple instances.
Sha1CheckSum s = RsDirUtil::sha1sum((uint8_t*)(fileInfo.filePath().toUtf8().constData()),fileInfo.filePath().toUtf8().size());
tree.addFile(dir_index,fileInfo.fileName().toUtf8().constData(),s,fileInfo.size());
paths_to_hash.insert(fileInfo.filePath().toUtf8().constData());
}
}
/**
@ -768,7 +780,6 @@ static void recursBuildFileTree(const QString& path,RsFileTree& tree,RsFileTree:
*/
void RsCollectionDialog::addSelection(bool recursive)
{
QStringList fileToHash;
QMap<QString, QString > dirToAdd;
int count=0;//to not scan all items on list .count()
@ -776,11 +787,13 @@ void RsCollectionDialog::addSelection(bool recursive)
mCollectionModel->preMods();
QSet<QString> paths_to_hash; // sha1sum of the paths to hash
foreach (QModelIndex index, milSelectionList)
if(index.column()==0) //Get only FileName
{
RsFileTree tree;
recursBuildFileTree(_dirModel->filePath(_tree_proxyModel->mapToSource(index)),tree,tree.root(),recursive);
recursBuildFileTree(_dirModel->filePath(_tree_proxyModel->mapToSource(index)),tree,tree.root(),recursive,paths_to_hash);
mCollection->merge_in(tree);
@ -852,9 +865,18 @@ void RsCollectionDialog::addSelection(bool recursive)
// Process Files once all done
ui._hashBox->addAttachments(fileToHash,RS_FILE_REQ_ANONYMOUS_ROUTING /*, 0*/);
#endif
// std::map<Sha1CheckSum,QString> paths_and_hashes;
// for(auto path:paths_to_hash)
// paths_and_hashes.insert(std::make_pair(RsDirUtil::sha1sum((uint8_t*)path.toUtf8().constData(),path.toUtf8().size()),path));
// mCollectionModel->addFilesToHash(paths_and_hashes);
mCollectionModel->postMods();
ui._hashBox->addAttachments(QStringList(paths_to_hash.begin(),paths_to_hash.end()),RS_FILE_REQ_ANONYMOUS_ROUTING /*, 0*/);
}
#ifdef TO_REMOVE
/**
* @brief RsCollectionDialog::addAllChild: Add children to RsCollection
* @param fileInfoParent: Parent's QFileInfo to scan
@ -910,6 +932,7 @@ bool RsCollectionDialog::addAllChild(QFileInfo &fileInfoParent
}
return true;
}
#endif
/**
* @brief RsCollectionDialog::remove: Remove selected Items in RSCollection
@ -1186,6 +1209,7 @@ void RsCollectionDialog::makeDir()
*/
void RsCollectionDialog::fileHashingFinished(QList<HashedFile> hashedFiles)
{
#ifdef TO_REMOVE
std::cerr << "RsCollectionDialog::fileHashingFinished() started." << std::endl;
QString message;
@ -1211,8 +1235,26 @@ void RsCollectionDialog::fileHashingFinished(QList<HashedFile> hashedFiles)
_newColFileInfos.push_back(colFileInfo);
#endif
}
#endif
// build a map of old-hash to new-hash for the hashed files, so that it can be passed to the mCollection for update
std::cerr << "RsCollectionDialog::fileHashingFinished message : " << message.toStdString() << std::endl;
std::map<RsFileHash,RsFileHash> old_to_new_hashes;
for(auto f:hashedFiles)
{
auto it = mFilesBeingHashed.find(f.filepath);
if(it == mFilesBeingHashed.end())
{
RsErr() << "Could not find hash-ID correspondence for path " << f.filepath.toUtf8().constData() << ". This is a bug." << std::endl;
continue;
}
old_to_new_hashes.insert(std::make_pair(it->second,f.hash));
mFilesBeingHashed.erase(it);
}
mCollectionModel->preMods();
mCollection->updateHashes(old_to_new_hashes);
mCollectionModel->postMods();
updateList();
}

View File

@ -18,6 +18,7 @@
* *
*******************************************************************************/
#include <set>
#include "ui_RsCollectionDialog.h"
#include "RsCollection.h"
#include "RsCollectionModel.h"
@ -90,12 +91,12 @@ private:
bool addChild(QTreeWidgetItem *parent, const std::vector<ColFileInfo> &child);
bool removeItem(QTreeWidgetItem *item, bool &removeOnlyFile) ;
void saveChild(QTreeWidgetItem *parentItem, ColFileInfo *parentInfo = NULL);
#endif
void addSelection(bool recursive) ;
bool addAllChild(QFileInfo &fileInfoParent
, QMap<QString, QString > &dirToAdd
, QStringList &fileToHash
, int &count);
#endif
void addSelection(bool recursive) ;
Ui::RsCollectionDialog ui;
QString _fileName ;
@ -110,4 +111,6 @@ private:
RsCollectionModel *mCollectionModel;
RsCollection *mCollection;
std::map<QString,RsFileHash> mFilesBeingHashed; // map of file path vs. temporary ID used for the file while hashing
};

View File

@ -252,9 +252,9 @@ bool RsCollectionModel::setData(const QModelIndex& index,const QVariant& value,i
if(role==Qt::CheckStateRole && convertInternalIdToIndex(index.internalId(), e))
{
//#ifdef DEBUG_COLLECTION_MODEL
#ifdef DEBUG_COLLECTION_MODEL
std::cerr << "Setting check state of item " << index << " to " << value.toBool() << std::endl;
//#endif
#endif
RsFileTree::DirIndex dir_index ;
if(e.is_file)
@ -370,7 +370,6 @@ QVariant RsCollectionModel::checkStateRole(const EntryIndex& i,int col) const
{
if(i.is_file)
{
std::cerr<< "entry is file, checkstate = " << (int)mFileInfos[i.index].is_checked << std::endl;
if(mFileInfos[i.index].is_checked)
return QVariant(Qt::Checked);
else
@ -378,8 +377,6 @@ QVariant RsCollectionModel::checkStateRole(const EntryIndex& i,int col) const
}
else
{
std::cerr<< "entry is dir, checkstate = " << (int)mDirInfos[i.index].check_state << std::endl;
switch(mDirInfos[i.index].check_state)
{
case SELECTED: return QVariant::fromValue((int)Qt::Checked);