mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-05-14 03:52:39 -04:00
Implement deep indexing for files through Xapian
ATM it support extracting metadata only from OGG files. The system has been designed to be easly extensible to more file formats registering more indexer functions which just need to extract metadata from a certain type of file and feed it to Xapian. The system has been integrated into existent file search system to through generric search requests and results, it keep a good level of retro-compatibility due to some tricks. The indexing system is released under AGPLv3 so when libretroshare is compiled with deep search enabled AGPLv3 must be honored instead of LGPLv3-or-later. Cleaned up the debian copyright file using non-deprecated license code-names.
This commit is contained in:
parent
d46e3eb2b7
commit
3a26ccf6a5
25 changed files with 1364 additions and 438 deletions
93
libretroshare/src/deep_search/commonutils.cpp
Normal file
93
libretroshare/src/deep_search/commonutils.cpp
Normal file
|
@ -0,0 +1,93 @@
|
|||
/*******************************************************************************
|
||||
* RetroShare full text indexing and search implementation based on Xapian *
|
||||
* *
|
||||
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
|
||||
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
|
||||
* *
|
||||
* This program is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the GNU Affero General Public License version 3 as *
|
||||
* published by the Free Software Foundation. *
|
||||
* *
|
||||
* This program is distributed in the hope that it will be useful, *
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
||||
* GNU Affero General Public License for more details. *
|
||||
* *
|
||||
* You should have received a copy of the GNU Affero General Public License *
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
|
||||
* *
|
||||
*******************************************************************************/
|
||||
|
||||
#include "deep_search/commonutils.hpp"
|
||||
#include "util/stacktrace.h"
|
||||
#include "util/rsdebug.h"
|
||||
|
||||
namespace DeepSearch
|
||||
{
|
||||
|
||||
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
|
||||
const std::string& path, int flags, int blockSize )
|
||||
{
|
||||
try
|
||||
{
|
||||
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
|
||||
new Xapian::WritableDatabase(path, flags, blockSize) );
|
||||
return dbPtr;
|
||||
}
|
||||
catch(Xapian::DatabaseLockError)
|
||||
{
|
||||
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
|
||||
<< path << std::endl;
|
||||
print_stacktrace();
|
||||
}
|
||||
catch(...)
|
||||
{
|
||||
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
|
||||
<< "deleting it might help without causing any harm: "
|
||||
<< path << std::endl;
|
||||
print_stacktrace();
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
|
||||
const std::string& path, int flags )
|
||||
{
|
||||
try
|
||||
{
|
||||
std::unique_ptr<Xapian::Database> dbPtr(
|
||||
new Xapian::Database(path, flags) );
|
||||
return dbPtr;
|
||||
}
|
||||
catch(Xapian::DatabaseOpeningError e)
|
||||
{
|
||||
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
|
||||
<< ", probably nothing has been indexed yet." << std::endl;
|
||||
}
|
||||
catch(Xapian::DatabaseLockError)
|
||||
{
|
||||
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
|
||||
<< path << std::endl;
|
||||
print_stacktrace();
|
||||
}
|
||||
catch(...)
|
||||
{
|
||||
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
|
||||
<< "deleting it might help without causing any harm: "
|
||||
<< path << std::endl;
|
||||
print_stacktrace();
|
||||
}
|
||||
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
std::string timetToXapianDate(const rstime_t& time)
|
||||
{
|
||||
char date[] = "YYYYMMDD\0";
|
||||
time_t tTime = static_cast<time_t>(time);
|
||||
std::strftime(date, 9, "%Y%m%d", std::gmtime(&tTime));
|
||||
return date;
|
||||
}
|
||||
|
||||
}
|
Loading…
Add table
Add a link
Reference in a new issue