Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-07-28 00:45:34 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement deep indexing for files through Xapian

Browse source 
ATM it support extracting metadata only from OGG files.
The system has been designed to be easly extensible to more file formats
  registering more indexer functions which just need to extract metadata
  from a certain type of file and feed it to Xapian.
The system has been integrated into existent file search system to
  through generric search requests and results, it keep a good level of
  retro-compatibility due to some tricks.
The indexing system is released under AGPLv3  so when libretroshare is compiled
  with deep search enabled AGPLv3 must be honored instead of LGPLv3-or-later.
Cleaned up the debian copyright file using non-deprecated license
  code-names.
This commit is contained in:
Gioacchino Mazzurco 2019-06-20 17:24:18 +02:00
parent d46e3eb2b7
commit 3a26ccf6a5
No known key found for this signature in database
GPG key ID: A1FBCA3872E87051
25 changed files with 1364 additions and 438 deletions
Download patch file Download diff file Expand all files Collapse all files

229
libretroshare/src/ft/ftserver.cc
View file

@ -54,6 +54,10 @@
#include <iostream>
#include "util/rstime.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
#endif // def RS_DEEP_FILES_INDEX
/***
* #define SERVER_DEBUG 1
* #define SERVER_DEBUG_CACHE 1
@ -65,9 +69,26 @@
static const rstime_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priority tasks handling every 5 seconds
static const rstime_t FILE_TRANSFER_MAX_DELAY_BEFORE_DROP_USAGE_RECORD = 10 ; // keep usage records for 10 secs at most.
#ifdef RS_DEEP_FILES_INDEX
TurtleFileInfoV2::TurtleFileInfoV2(const DeepFilesSearchResult& dRes) :
fHash(dRes.mFileHash), fWeight(static_cast<float>(dRes.mWeight)),
fSnippet(dRes.mSnippet)
{
FileInfo fInfo;
rsFiles->FileDetails(fHash, RS_FILE_HINTS_LOCAL, fInfo);
fSize = fInfo.size;
fName = fInfo.fname;
}
#endif // def RS_DEEP_FILES_INDEX
TurtleFileInfoV2::~TurtleFileInfoV2() = default;
/* Setup */
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc)
: p3Service(),RsServiceSerializer(RS_SERVICE_TYPE_TURTLE), // should be FT, but this is for backward compatibility
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc):
p3Service(),
// should be FT, but this is for backward compatibility
RsServiceSerializer(RS_SERVICE_TYPE_TURTLE),
mPeerMgr(pm), mServiceCtrl(sc),
mFileDatabase(NULL),
mFtController(NULL), mFtExtra(NULL),
@ -500,15 +521,24 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi
return false;
}
RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const
RsItem *ftServer::create_item(uint16_t service, uint8_t item_type) const
{
#ifdef SERVER_DEBUG
FTSERVER_DEBUG() << "p3turtle: deserialising packet: " << std::endl ;
#endif
if (RS_SERVICE_TYPE_TURTLE != service)
RsServiceType serviceType = static_cast<RsServiceType>(service);
switch (serviceType)
{
FTSERVER_ERROR() << " Wrong type !!" << std::endl ;
return NULL; /* wrong type */
/* This one is here for retro-compatibility as turtle routing and file
* trasfer services were just one service before turle service was
* generalized */
case RsServiceType::TURTLE: break;
case RsServiceType::FILE_TRANSFER: break;
default:
RsErr() << __PRETTY_FUNCTION__ << " Wrong service type: " << service
<< std::endl;
return nullptr;
}
try
@ -521,16 +551,19 @@ RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem();
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST : return new RsTurtleChunkCrcRequestItem();
case RS_TURTLE_SUBTYPE_CHUNK_CRC : return new RsTurtleChunkCrcItem();
case static_cast<uint8_t>(RsFileItemType::FILE_SEARCH_REQUEST):
return new RsFileSearchRequestItem();
case static_cast<uint8_t>(RsFileItemType::FILE_SEARCH_RESULT):
return new RsFileSearchResultItem();
default:
return NULL ;
return nullptr;
}
}
catch(std::exception& e)
{
FTSERVER_ERROR() << "(EE) deserialisation error in " << __PRETTY_FUNCTION__ << ": " << e.what() << std::endl;
return NULL ;
return nullptr;
}
}
@ -1837,7 +1870,12 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item)
if(cbpt != mSearchCallbacksMap.end())
{
hasCallback = true;
cbpt->second.first(item->result);
std::vector<TurtleFileInfoV2> cRes;
for( const auto& tfiold : item->result)
cRes.push_back(tfiold);
cbpt->second.first(cRes);
}
} // end RS_STACK_MUTEX(mSearchCallbacksMapMutex);
@ -1845,6 +1883,99 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item)
RsServer::notify()->notifyTurtleSearchResult(item->PeerId(),item->request_id, item->result );
}
bool ftServer::receiveSearchRequest(
unsigned char* searchRequestData, uint32_t searchRequestDataLen,
unsigned char*& searchResultData, uint32_t& searchResultDataLen,
uint32_t& maxAllowsHits )
{
#ifdef RS_DEEP_FILES_INDEX
std::unique_ptr<RsItem> recvItem(
RsServiceSerializer::deserialise(
searchRequestData, &searchRequestDataLen ) );
if(!recvItem)
{
RsWarn() << __PRETTY_FUNCTION__ << " Search request deserialization "
<< "failed" << std::endl;
return false;
}
std::unique_ptr<RsFileSearchRequestItem> sReqItPtr(
dynamic_cast<RsFileSearchRequestItem*>(recvItem.get()) );
if(!sReqItPtr)
{
RsWarn() << __PRETTY_FUNCTION__ << " Received an invalid search request"
<< " " << *recvItem << std::endl;
return false;
}
recvItem.release();
RsFileSearchRequestItem& searchReq(*sReqItPtr);
std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
{
RsFileSearchResultItem resIt;
for(const auto& dMatch : dRes)
resIt.mResults.push_back(TurtleFileInfoV2(dMatch));
searchResultDataLen = RsServiceSerializer::size(&resIt);
searchResultData = static_cast<uint8_t*>(malloc(searchResultDataLen));
return RsServiceSerializer::serialise(
&resIt, searchResultData, &searchResultDataLen );
}
#endif // def RS_DEEP_FILES_INDEX
searchResultData = nullptr;
searchResultDataLen = 0;
return false;
}
void ftServer::receiveSearchResult(
TurtleSearchRequestId requestId, unsigned char* searchResultData,
uint32_t searchResultDataLen )
{
if(!searchResultData || !searchResultDataLen)
{
RsWarn() << __PRETTY_FUNCTION__ << " got null paramethers "
<< "searchResultData: " << static_cast<void*>(searchResultData)
<< " searchResultDataLen: " << searchResultDataLen
<< " seems someone else in the network have a buggy RetroShare"
<< " implementation" << std::endl;
return;
}
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
auto cbpt = mSearchCallbacksMap.find(requestId);
if(cbpt != mSearchCallbacksMap.end())
{
RsItem* recvItem = RsServiceSerializer::deserialise(
searchResultData, &searchResultDataLen );
if(!recvItem)
{
RsWarn() << __PRETTY_FUNCTION__ << " Search result deserialization "
<< "failed" << std::endl;
return;
}
std::unique_ptr<RsFileSearchResultItem> resItPtr(
dynamic_cast<RsFileSearchResultItem*>(recvItem) );
if(!resItPtr)
{
RsWarn() << __PRETTY_FUNCTION__ << " Received invalid search result"
<< std::endl;
delete recvItem;
return;
}
cbpt->second.first(resItPtr->mResults);
}
}
/***************************** CONFIG ****************************/
bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
@ -1857,27 +1988,74 @@ bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
return true;
}
#ifdef RS_DEEP_FILES_INDEX
static std::vector<std::string> xapianQueryKeywords =
{
" AND ", " OR ", " NOT ", " XOR ", " +", " -", " ( ", " ) ", " NEAR ",
" ADJ ", " \"", "\" "
};
#endif
bool ftServer::turtleSearchRequest(
const std::string& matchString,
const std::function<void (const std::list<TurtleFileInfo>& results)>& multiCallback,
const std::function<void (const std::vector<TurtleFileInfoV2>& results)>& multiCallback,
rstime_t maxWait )
{
if(matchString.empty())
{
std::cerr << __PRETTY_FUNCTION__ << " match string can't be empty!"
<< std::endl;
RsWarn() << __PRETTY_FUNCTION__ << " match string can't be empty!"
<< std::endl;
return false;
}
TurtleRequestId sId = turtleSearch(matchString);
#ifdef RS_DEEP_FILES_INDEX
RsFileSearchRequestItem sItem;
sItem.queryString = matchString;
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
mSearchCallbacksMap.emplace(
sId,
std::make_pair(
multiCallback,
std::chrono::system_clock::now() +
std::chrono::seconds(maxWait) ) );
uint32_t iSize = RsServiceSerializer::size(&sItem);
uint8_t* iBuf = static_cast<uint8_t*>(malloc(iSize));
RsServiceSerializer::serialise(&sItem, iBuf, &iSize);
Dbg3() << __PRETTY_FUNCTION__ << " sending search request:" << sItem
<< std::endl;
TurtleRequestId xsId = mTurtleRouter->turtleSearch(iBuf, iSize, this);
{ RS_STACK_MUTEX(mSearchCallbacksMapMutex);
mSearchCallbacksMap.emplace(
xsId,
std::make_pair(
multiCallback,
std::chrono::system_clock::now() +
std::chrono::seconds(maxWait) ) );
} // RS_STACK_MUTEX(mSearchCallbacksMapMutex);
/* Trick to keep receiving more or less usable results from old peers */
std::string strippedQuery = matchString;
for(const std::string& xKeyword : xapianQueryKeywords)
{
std::string::size_type pos = std::string::npos;
while( (pos = strippedQuery.find(xKeyword)) != std::string::npos )
strippedQuery.replace(pos, xKeyword.length(), " ");
}
Dbg3() << __PRETTY_FUNCTION__ << " sending stripped query for "
<< "retro-compatibility: " << strippedQuery << std::endl;
TurtleRequestId sId = mTurtleRouter->turtleSearch(strippedQuery);
#else // def RS_DEEP_FILES_INDEX
TurtleRequestId sId = mTurtleRouter->turtleSearch(matchString);
#endif // def RS_DEEP_FILES_INDEX
{
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
mSearchCallbacksMap.emplace(
sId,
std::make_pair(
multiCallback,
std::chrono::system_clock::now() +
std::chrono::seconds(maxWait) ) );
}
return true;
}
@ -1913,4 +2091,13 @@ bool ftServer::isHashBanned(const RsFileHash& hash)
return mFileDatabase->isFileBanned(hash);
}
RsFileItem::~RsFileItem() = default;
RsFileItem::RsFileItem(RsFileItemType subtype) :
RsItem( RS_PKT_VERSION_SERVICE,
static_cast<uint16_t>(RsServiceType::FILE_TRANSFER),
static_cast<uint8_t>(subtype) ) {}
void RsFileSearchRequestItem::clear() { queryString.clear(); }
void RsFileSearchResultItem::clear() { mResults.clear(); }

73
libretroshare/src/ft/ftserver.h
View file

@ -46,7 +46,8 @@
#include "turtle/turtleclientservice.h"
#include "services/p3service.h"
#include "retroshare/rsfiles.h"
#include "rsitems/rsitem.h"
#include "serialiser/rsserial.h"
#include "pqi/pqi.h"
#include "pqi/p3cfgmgr.h"
@ -67,7 +68,53 @@ class p3PeerMgr;
class p3ServiceControl;
class p3FileDatabase;
class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService, public RsServiceSerializer
enum class RsFileItemType : uint8_t
{
NONE = 0x00, /// Only to detect ununitialized
FILE_SEARCH_REQUEST = 0x57,
FILE_SEARCH_RESULT = 0x58
};
struct RsFileItem : RsItem
{
~RsFileItem() override;
protected:
RsFileItem(RsFileItemType subtype);
};
struct RsFileSearchRequestItem : RsFileItem
{
RsFileSearchRequestItem() : RsFileItem(RsFileItemType::FILE_SEARCH_REQUEST)
{ setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_REQUEST); }
std::string queryString;
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{ RS_SERIAL_PROCESS(queryString); }
void clear() override;
};
struct RsFileSearchResultItem : RsFileItem
{
RsFileSearchResultItem() : RsFileItem(RsFileItemType::FILE_SEARCH_RESULT)
{ setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT); }
std::vector<TurtleFileInfoV2> mResults;
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{ RS_SERIAL_PROCESS(mResults); }
void clear() override;
};
class ftServer :
public p3Service, public RsFiles, public ftDataSend,
public RsTurtleClientService, public RsServiceSerializer
{
public:
@ -98,7 +145,21 @@ public:
uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ;
virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
virtual void ftReceiveSearchResult(RsTurtleFTSearchResultItem *item); // We dont use TurtleClientService::receiveSearchResult() because of backward compatibility.
/// We keep this for retro-compatibility @see RsTurtleClientService
virtual void ftReceiveSearchResult(RsTurtleFTSearchResultItem *item);
/// @see RsTurtleClientService
bool receiveSearchRequest(
unsigned char* searchRequestData, uint32_t searchRequestDataLen,
unsigned char*& search_result_data, uint32_t& searchResultDataLen,
uint32_t& maxAllowsHits ) override;
/// @see RsTurtleClientService
void receiveSearchResult(
TurtleSearchRequestId requestId, unsigned char* searchResultData,
uint32_t searchResultDataLen ) override;
virtual RsItem *create_item(uint16_t service,uint8_t item_type) const ;
virtual RsServiceSerializer *serializer() { return this ; }
@ -148,7 +209,7 @@ public:
/// @see RsFiles
virtual bool turtleSearchRequest(
const std::string& matchString,
const std::function<void (const std::list<TurtleFileInfo>& results)>& multiCallback,
const std::function<void (const std::vector<TurtleFileInfoV2>& results)>& multiCallback,
rstime_t maxWait = 300 );
virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
@ -337,13 +398,15 @@ private:
std::map<
TurtleRequestId,
std::pair<
std::function<void (const std::list<TurtleFileInfo>& results)>,
std::function<void (const std::vector<TurtleFileInfoV2>& results)>,
std::chrono::system_clock::time_point >
> mSearchCallbacksMap;
RsMutex mSearchCallbacksMapMutex;
/// Cleanup mSearchCallbacksMap
void cleanTimedOutSearches();
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};