Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-05-11 10:35:22 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement deep indexing for files through Xapian

Browse source 
ATM it support extracting metadata only from OGG files.
The system has been designed to be easly extensible to more file formats
  registering more indexer functions which just need to extract metadata
  from a certain type of file and feed it to Xapian.
The system has been integrated into existent file search system to
  through generric search requests and results, it keep a good level of
  retro-compatibility due to some tricks.
The indexing system is released under AGPLv3  so when libretroshare is compiled
  with deep search enabled AGPLv3 must be honored instead of LGPLv3-or-later.
Cleaned up the debian copyright file using non-deprecated license
  code-names.
This commit is contained in:
Gioacchino Mazzurco 2019-06-20 17:24:18 +02:00
parent d46e3eb2b7
commit 3a26ccf6a5
No known key found for this signature in database
GPG key ID: A1FBCA3872E87051
25 changed files with 1364 additions and 438 deletions
Download patch file Download diff file Expand all files Collapse all files

15
libretroshare/src/file_sharing/dir_hierarchy.cc
View file

@ -21,15 +21,19 @@
******************************************************************************/
#include <sstream>
#include <algorithm>
#include "util/rstime.h"
#include "util/rsdir.h"
#include "util/rsprint.h"
#include "retroshare/rsexpr.h"
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "file_sharing_defaults.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
#endif // def RS_DEEP_FILES_INDEX
//#define DEBUG_DIRECTORY_STORAGE 1
typedef FileListIO::read_error read_error;
@ -391,6 +395,11 @@ void InternalFileHierarchyStorage::deleteFileNode(uint32_t index)
{
FileEntry& fe(*static_cast<FileEntry*>(mNodes[index])) ;
#ifdef RS_DEEP_FILES_INDEX
DeepFilesIndex tfi(DeepFilesIndex::dbDefaultPath());
tfi.removeFileFromIndex(fe.file_hash);
#endif
if(mTotalSize >= fe.file_size)
mTotalSize -= fe.file_size ;
@ -753,7 +762,9 @@ int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression
return 0;
}
int InternalFileHierarchyStorage::searchTerms(const std::list<std::string>& terms, std::list<DirectoryStorage::EntryIndex> &results) const
int InternalFileHierarchyStorage::searchTerms(
const std::list<std::string>& terms,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
// most entries are likely to be files, so we could do a linear search over the entries tab.
// instead we go through the table of hashes.

44
libretroshare/src/file_sharing/directory_storage.cc
View file

@ -20,6 +20,7 @@
* *
******************************************************************************/
#include <set>
#include "util/rstime.h"
#include "serialiser/rstlvbinary.h"
#include "retroshare/rspeers.h"
@ -30,6 +31,10 @@
#include "dir_hierarchy.h"
#include "filelist_io.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
#endif // def RS_DEEP_FILES_INDEX
//#define DEBUG_REMOTE_DIRECTORY_STORAGE 1
/******************************************************************************************************************/
@ -180,7 +185,9 @@ void DirectoryStorage::print()
mFileHierarchy->print();
}
int DirectoryStorage::searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const
int DirectoryStorage::searchTerms(
const std::list<std::string>& terms,
std::list<EntryIndex>& results ) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchTerms(terms,results);
@ -501,18 +508,39 @@ void LocalDirectoryStorage::updateTimeStamps()
#endif
}
}
bool LocalDirectoryStorage::updateHash(const EntryIndex& index, const RsFileHash& hash, bool update_internal_hierarchy)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mEncryptedHashes[makeEncryptedHash(hash)] = hash ;
mChanged = true ;
bool LocalDirectoryStorage::updateHash(
const EntryIndex& index, const RsFileHash& hash,
bool update_internal_hierarchy )
{
bool ret = false;
{
RS_STACK_MUTEX(mDirStorageMtx);
mEncryptedHashes[makeEncryptedHash(hash)] = hash ;
mChanged = true ;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Updating index of hash " << hash << " update_internal=" << update_internal_hierarchy << std::endl;
std::cerr << "Updating index of hash " << hash << " update_internal="
<< update_internal_hierarchy << std::endl;
#endif
return (!update_internal_hierarchy)|| mFileHierarchy->updateHash(index,hash);
ret = (!update_internal_hierarchy) ||
mFileHierarchy->updateHash(index,hash);
} // RS_STACK_MUTEX(mDirStorageMtx);
#ifdef RS_DEEP_FILES_INDEX
FileInfo fInfo;
if( ret && getFileInfo(index, fInfo) &&
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
}
#endif // def RS_DEEP_FILES_INDEX
return ret;
}
std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const
{