mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-11-25 18:16:30 -05:00
Implement deep indexing for files through Xapian
ATM it support extracting metadata only from OGG files. The system has been designed to be easly extensible to more file formats registering more indexer functions which just need to extract metadata from a certain type of file and feed it to Xapian. The system has been integrated into existent file search system to through generric search requests and results, it keep a good level of retro-compatibility due to some tricks. The indexing system is released under AGPLv3 so when libretroshare is compiled with deep search enabled AGPLv3 must be honored instead of LGPLv3-or-later. Cleaned up the debian copyright file using non-deprecated license code-names.
This commit is contained in:
Gioacchino Mazzurco
parent
d46e3eb2b7
commit
3a26ccf6a5
25 changed files with 1364 additions and 438 deletions
|
|
@ -54,6 +54,10 @@
|
|||
#include <iostream>
|
||||
#include "util/rstime.h"
|
||||
|
||||
#ifdef RS_DEEP_FILES_INDEX
|
||||
# include "deep_search/filesindex.hpp"
|
||||
#endif // def RS_DEEP_FILES_INDEX
|
||||
|
||||
/***
|
||||
* #define SERVER_DEBUG 1
|
||||
* #define SERVER_DEBUG_CACHE 1
|
||||
|
|
@ -65,9 +69,26 @@
|
|||
static const rstime_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priority tasks handling every 5 seconds
|
||||
static const rstime_t FILE_TRANSFER_MAX_DELAY_BEFORE_DROP_USAGE_RECORD = 10 ; // keep usage records for 10 secs at most.
|
||||
|
||||
#ifdef RS_DEEP_FILES_INDEX
|
||||
TurtleFileInfoV2::TurtleFileInfoV2(const DeepFilesSearchResult& dRes) :
|
||||
fHash(dRes.mFileHash), fWeight(static_cast<float>(dRes.mWeight)),
|
||||
fSnippet(dRes.mSnippet)
|
||||
{
|
||||
FileInfo fInfo;
|
||||
rsFiles->FileDetails(fHash, RS_FILE_HINTS_LOCAL, fInfo);
|
||||
|
||||
fSize = fInfo.size;
|
||||
fName = fInfo.fname;
|
||||
}
|
||||
#endif // def RS_DEEP_FILES_INDEX
|
||||
|
||||
TurtleFileInfoV2::~TurtleFileInfoV2() = default;
|
||||
|
||||
/* Setup */
|
||||
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc)
|
||||
: p3Service(),RsServiceSerializer(RS_SERVICE_TYPE_TURTLE), // should be FT, but this is for backward compatibility
|
||||
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc):
|
||||
p3Service(),
|
||||
// should be FT, but this is for backward compatibility
|
||||
RsServiceSerializer(RS_SERVICE_TYPE_TURTLE),
|
||||
mPeerMgr(pm), mServiceCtrl(sc),
|
||||
mFileDatabase(NULL),
|
||||
mFtController(NULL), mFtExtra(NULL),
|
||||
|
|
@ -500,15 +521,24 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi
|
|||
return false;
|
||||
}
|
||||
|
||||
RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const
|
||||
RsItem *ftServer::create_item(uint16_t service, uint8_t item_type) const
|
||||
{
|
||||
#ifdef SERVER_DEBUG
|
||||
FTSERVER_DEBUG() << "p3turtle: deserialising packet: " << std::endl ;
|
||||
#endif
|
||||
if (RS_SERVICE_TYPE_TURTLE != service)
|
||||
|
||||
RsServiceType serviceType = static_cast<RsServiceType>(service);
|
||||
switch (serviceType)
|
||||
{
|
||||
FTSERVER_ERROR() << " Wrong type !!" << std::endl ;
|
||||
return NULL; /* wrong type */
|
||||
/* This one is here for retro-compatibility as turtle routing and file
|
||||
* trasfer services were just one service before turle service was
|
||||
* generalized */
|
||||
case RsServiceType::TURTLE: break;
|
||||
case RsServiceType::FILE_TRANSFER: break;
|
||||
default:
|
||||
RsErr() << __PRETTY_FUNCTION__ << " Wrong service type: " << service
|
||||
<< std::endl;
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
try
|
||||
|
|
@ -521,16 +551,19 @@ RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const
|
|||
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem();
|
||||
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST : return new RsTurtleChunkCrcRequestItem();
|
||||
case RS_TURTLE_SUBTYPE_CHUNK_CRC : return new RsTurtleChunkCrcItem();
|
||||
|
||||
case static_cast<uint8_t>(RsFileItemType::FILE_SEARCH_REQUEST):
|
||||
return new RsFileSearchRequestItem();
|
||||
case static_cast<uint8_t>(RsFileItemType::FILE_SEARCH_RESULT):
|
||||
return new RsFileSearchResultItem();
|
||||
default:
|
||||
return NULL ;
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
catch(std::exception& e)
|
||||
{
|
||||
FTSERVER_ERROR() << "(EE) deserialisation error in " << __PRETTY_FUNCTION__ << ": " << e.what() << std::endl;
|
||||
|
||||
return NULL ;
|
||||
return nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1837,7 +1870,12 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item)
|
|||
if(cbpt != mSearchCallbacksMap.end())
|
||||
{
|
||||
hasCallback = true;
|
||||
cbpt->second.first(item->result);
|
||||
|
||||
std::vector<TurtleFileInfoV2> cRes;
|
||||
for( const auto& tfiold : item->result)
|
||||
cRes.push_back(tfiold);
|
||||
|
||||
cbpt->second.first(cRes);
|
||||
}
|
||||
} // end RS_STACK_MUTEX(mSearchCallbacksMapMutex);
|
||||
|
||||
|
|
@ -1845,6 +1883,99 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item)
|
|||
RsServer::notify()->notifyTurtleSearchResult(item->PeerId(),item->request_id, item->result );
|
||||
}
|
||||
|
||||
bool ftServer::receiveSearchRequest(
|
||||
unsigned char* searchRequestData, uint32_t searchRequestDataLen,
|
||||
unsigned char*& searchResultData, uint32_t& searchResultDataLen,
|
||||
uint32_t& maxAllowsHits )
|
||||
{
|
||||
#ifdef RS_DEEP_FILES_INDEX
|
||||
std::unique_ptr<RsItem> recvItem(
|
||||
RsServiceSerializer::deserialise(
|
||||
searchRequestData, &searchRequestDataLen ) );
|
||||
|
||||
if(!recvItem)
|
||||
{
|
||||
RsWarn() << __PRETTY_FUNCTION__ << " Search request deserialization "
|
||||
<< "failed" << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
std::unique_ptr<RsFileSearchRequestItem> sReqItPtr(
|
||||
dynamic_cast<RsFileSearchRequestItem*>(recvItem.get()) );
|
||||
if(!sReqItPtr)
|
||||
{
|
||||
RsWarn() << __PRETTY_FUNCTION__ << " Received an invalid search request"
|
||||
<< " " << *recvItem << std::endl;
|
||||
return false;
|
||||
}
|
||||
recvItem.release();
|
||||
|
||||
RsFileSearchRequestItem& searchReq(*sReqItPtr);
|
||||
|
||||
std::vector<DeepFilesSearchResult> dRes;
|
||||
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
|
||||
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
|
||||
{
|
||||
RsFileSearchResultItem resIt;
|
||||
|
||||
for(const auto& dMatch : dRes)
|
||||
resIt.mResults.push_back(TurtleFileInfoV2(dMatch));
|
||||
|
||||
searchResultDataLen = RsServiceSerializer::size(&resIt);
|
||||
searchResultData = static_cast<uint8_t*>(malloc(searchResultDataLen));
|
||||
return RsServiceSerializer::serialise(
|
||||
&resIt, searchResultData, &searchResultDataLen );
|
||||
}
|
||||
#endif // def RS_DEEP_FILES_INDEX
|
||||
|
||||
searchResultData = nullptr;
|
||||
searchResultDataLen = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
void ftServer::receiveSearchResult(
|
||||
TurtleSearchRequestId requestId, unsigned char* searchResultData,
|
||||
uint32_t searchResultDataLen )
|
||||
{
|
||||
if(!searchResultData || !searchResultDataLen)
|
||||
{
|
||||
RsWarn() << __PRETTY_FUNCTION__ << " got null paramethers "
|
||||
<< "searchResultData: " << static_cast<void*>(searchResultData)
|
||||
<< " searchResultDataLen: " << searchResultDataLen
|
||||
<< " seems someone else in the network have a buggy RetroShare"
|
||||
<< " implementation" << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
|
||||
auto cbpt = mSearchCallbacksMap.find(requestId);
|
||||
if(cbpt != mSearchCallbacksMap.end())
|
||||
{
|
||||
RsItem* recvItem = RsServiceSerializer::deserialise(
|
||||
searchResultData, &searchResultDataLen );
|
||||
|
||||
if(!recvItem)
|
||||
{
|
||||
RsWarn() << __PRETTY_FUNCTION__ << " Search result deserialization "
|
||||
<< "failed" << std::endl;
|
||||
return;
|
||||
}
|
||||
|
||||
std::unique_ptr<RsFileSearchResultItem> resItPtr(
|
||||
dynamic_cast<RsFileSearchResultItem*>(recvItem) );
|
||||
|
||||
if(!resItPtr)
|
||||
{
|
||||
RsWarn() << __PRETTY_FUNCTION__ << " Received invalid search result"
|
||||
<< std::endl;
|
||||
delete recvItem;
|
||||
return;
|
||||
}
|
||||
|
||||
cbpt->second.first(resItPtr->mResults);
|
||||
}
|
||||
}
|
||||
|
||||
/***************************** CONFIG ****************************/
|
||||
|
||||
bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
|
||||
|
|
@ -1857,27 +1988,74 @@ bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
|
|||
return true;
|
||||
}
|
||||
|
||||
#ifdef RS_DEEP_FILES_INDEX
|
||||
static std::vector<std::string> xapianQueryKeywords =
|
||||
{
|
||||
" AND ", " OR ", " NOT ", " XOR ", " +", " -", " ( ", " ) ", " NEAR ",
|
||||
" ADJ ", " \"", "\" "
|
||||
};
|
||||
#endif
|
||||
|
||||
bool ftServer::turtleSearchRequest(
|
||||
const std::string& matchString,
|
||||
const std::function<void (const std::list<TurtleFileInfo>& results)>& multiCallback,
|
||||
const std::function<void (const std::vector<TurtleFileInfoV2>& results)>& multiCallback,
|
||||
rstime_t maxWait )
|
||||
{
|
||||
if(matchString.empty())
|
||||
{
|
||||
std::cerr << __PRETTY_FUNCTION__ << " match string can't be empty!"
|
||||
<< std::endl;
|
||||
RsWarn() << __PRETTY_FUNCTION__ << " match string can't be empty!"
|
||||
<< std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
TurtleRequestId sId = turtleSearch(matchString);
|
||||
#ifdef RS_DEEP_FILES_INDEX
|
||||
RsFileSearchRequestItem sItem;
|
||||
sItem.queryString = matchString;
|
||||
|
||||
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
|
||||
mSearchCallbacksMap.emplace(
|
||||
sId,
|
||||
std::make_pair(
|
||||
multiCallback,
|
||||
std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(maxWait) ) );
|
||||
uint32_t iSize = RsServiceSerializer::size(&sItem);
|
||||
uint8_t* iBuf = static_cast<uint8_t*>(malloc(iSize));
|
||||
RsServiceSerializer::serialise(&sItem, iBuf, &iSize);
|
||||
|
||||
Dbg3() << __PRETTY_FUNCTION__ << " sending search request:" << sItem
|
||||
<< std::endl;
|
||||
|
||||
TurtleRequestId xsId = mTurtleRouter->turtleSearch(iBuf, iSize, this);
|
||||
|
||||
{ RS_STACK_MUTEX(mSearchCallbacksMapMutex);
|
||||
mSearchCallbacksMap.emplace(
|
||||
xsId,
|
||||
std::make_pair(
|
||||
multiCallback,
|
||||
std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(maxWait) ) );
|
||||
} // RS_STACK_MUTEX(mSearchCallbacksMapMutex);
|
||||
|
||||
/* Trick to keep receiving more or less usable results from old peers */
|
||||
std::string strippedQuery = matchString;
|
||||
for(const std::string& xKeyword : xapianQueryKeywords)
|
||||
{
|
||||
std::string::size_type pos = std::string::npos;
|
||||
while( (pos = strippedQuery.find(xKeyword)) != std::string::npos )
|
||||
strippedQuery.replace(pos, xKeyword.length(), " ");
|
||||
}
|
||||
|
||||
Dbg3() << __PRETTY_FUNCTION__ << " sending stripped query for "
|
||||
<< "retro-compatibility: " << strippedQuery << std::endl;
|
||||
|
||||
TurtleRequestId sId = mTurtleRouter->turtleSearch(strippedQuery);
|
||||
#else // def RS_DEEP_FILES_INDEX
|
||||
TurtleRequestId sId = mTurtleRouter->turtleSearch(matchString);
|
||||
#endif // def RS_DEEP_FILES_INDEX
|
||||
|
||||
{
|
||||
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
|
||||
mSearchCallbacksMap.emplace(
|
||||
sId,
|
||||
std::make_pair(
|
||||
multiCallback,
|
||||
std::chrono::system_clock::now() +
|
||||
std::chrono::seconds(maxWait) ) );
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
|
@ -1913,4 +2091,13 @@ bool ftServer::isHashBanned(const RsFileHash& hash)
|
|||
return mFileDatabase->isFileBanned(hash);
|
||||
}
|
||||
|
||||
RsFileItem::~RsFileItem() = default;
|
||||
|
||||
RsFileItem::RsFileItem(RsFileItemType subtype) :
|
||||
RsItem( RS_PKT_VERSION_SERVICE,
|
||||
static_cast<uint16_t>(RsServiceType::FILE_TRANSFER),
|
||||
static_cast<uint8_t>(subtype) ) {}
|
||||
|
||||
void RsFileSearchRequestItem::clear() { queryString.clear(); }
|
||||
|
||||
void RsFileSearchResultItem::clear() { mResults.clear(); }
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue