Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #27 from RetroShare/master

Browse Source 
update to lastest master
This commit is contained in:
defnax 2019-11-18 17:03:50 +01:00 committed by GitHub
commit 7cb4afee84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
77 changed files with 2744 additions and 3012 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build_scripts/Android/prepare-toolchain-clang.sh
View File

@ -424,7 +424,7 @@ build_restbed()
rm -rf restbed-build; mkdir restbed-build ; cd restbed-build
cmake \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
-DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_TESTS=OFF \
-DBUILD_SSL=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../restbed
make -j${HOST_NUM_CPU}
make install

53
build_scripts/Debian/debian/copyright
View File

@ -1,19 +1,23 @@
Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
Upstream-Name: retroshare
Upstream-Contact: retroshare.team@gmail.com
Upstream-Contact: contact@retroshare.cc
Source: https://github.com/retroshare/retroshare
Files: openpgpsdk/*
Copyright: 2005-2008 Ben Laurie, Rachel Willmer, Retroshare Team <retroshare.team@gmail.com>
Copyright: 2005-2008 Ben Laurie, Rachel Willmer, Retroshare Team <contact@retroshare.cc>
License: Apache-2.0
Files: jsonapi-generator/* libretroshare/src/jsonapi/*
Copyright: 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org>
License: AGPL-3+
License: AGPL-3.0-or-later
Files: libretroshare/src/deep_search/*
Copyright: 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org>
License: AGPL-3.0-only
Files: libretroshare/*
Copyright: 2007-2018, Retroshare Team <retroshare.team@gmail.com>
License: LGPL-3+
Copyright: 2007-2019, Retroshare Team <contact@retroshare.cc>
License: LGPL-3.0-or-later
Files: src/retroshare-gui/src/TorControl/
Copyright: 2014, John Brooks <john.brooks@dereferenced.net>
@ -28,8 +32,8 @@ Copyright: 2013 Jeff Weinstein <jeff.weinstein@gmail.com>
License: MIT
Files: *
Copyright: 2007-2018, Retroshare Team <retroshare.team@gmail.com>
License: AGPL-3+
Copyright: 2007-2019, Retroshare Team <contact@retroshare.cc>
License: AGPL-3.0-only
#######
# TODO
@ -56,7 +60,7 @@ License: Apache-2.0
See the License for the specific language governing permissions and
limitations under the License.
License: LGPL-3+
License: LGPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Lesser General Public License as
published by the Free Software Foundation, either version 3 of the
@ -75,7 +79,7 @@ License: LGPL-3+
OpenSSL that use the same license as OpenSSL), and distribute linked
combinations including the two.
License: AGPL-3+
License: AGPL-3.0-or-later
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
@ -86,11 +90,36 @@ License: AGPL-3+
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
.
You should have received a copy of the GNU Lesser General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
.
As a special exception, the copyright holders give permission to link the
code of portions of this program with the OpenSSL library under certain
code or portions of this program with the OpenSSL library under certain
conditions as described in each individual source file and distribute
linked combinations including the program with the OpenSSL library. You
must comply with the GNU Affero General Public License in all respects for
all of the code used other than as permitted herein. If you modify file(s)
with this exception, you may extend this exception to your version of the
file(s), but you are not obligated to do so. If you do not wish to do so,
delete this exception statement from your version. If you delete this
exception statement from all source files in the program, then also delete
it in the license file.
License: AGPL-3.0-only
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License version 3 as
published by the Free Software Foundation.
.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Lesser General Public License for more details.
.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
.
As a special exception, the copyright holders give permission to link the
code or portions of this program with the OpenSSL library under certain
conditions as described in each individual source file and distribute
linked combinations including the program with the OpenSSL library. You
must comply with the GNU Affero General Public License in all respects for

20
libretroshare/src/chat/distantchat.cc
View File

@ -178,8 +178,7 @@ bool DistantChatService::acceptDataFromPeer(const RsGxsId& gxs_id,const RsGxsTun
return res ;
}
void DistantChatService::notifyTunnelStatus(
const RsGxsTunnelId& tunnel_id, uint32_t tunnel_status )
void DistantChatService::notifyTunnelStatus( const RsGxsTunnelId& tunnel_id, uint32_t tunnel_status )
{
#ifdef DEBUG_DISTANT_CHAT
DISTANT_CHAT_DEBUG() << "DistantChatService::notifyTunnelStatus(): got notification " << std::hex << tunnel_status << std::dec << " for tunnel " << tunnel_id << std::endl;
@ -195,18 +194,17 @@ void DistantChatService::notifyTunnelStatus(
RsServer::notify()->notifyPeerStatusChanged(tunnel_id.toStdString(),RS_STATUS_ONLINE) ;
break ;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"tunnel is down...") ;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"Tunnel is down...") ;
RsServer::notify()->notifyPeerStatusChanged(tunnel_id.toStdString(),RS_STATUS_OFFLINE) ;
break ;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"tunnel is down...") ;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"Tunnel is down...") ;
RsServer::notify()->notifyPeerStatusChanged(tunnel_id.toStdString(),RS_STATUS_OFFLINE) ;
break ;
}
}
void DistantChatService::receiveData(
const RsGxsTunnelId& tunnel_id, unsigned char* data, uint32_t data_size)
void DistantChatService::receiveData( const RsGxsTunnelId& tunnel_id, unsigned char* data, uint32_t data_size)
{
#ifdef DEBUG_DISTANT_CHAT
DISTANT_CHAT_DEBUG() << "DistantChatService::receiveData(): got data of size " << std::dec << data_size << " for tunnel " << tunnel_id << std::endl;
@ -303,16 +301,14 @@ bool DistantChatService::getDistantChatStatus(const DistantChatPeerId& tunnel_id
cinfo.to_id = tinfo.destination_gxs_id;
cinfo.own_id = tinfo.source_gxs_id;
cinfo.pending_items = tinfo.pending_data_packets;
cinfo.peer_id = tunnel_id;
switch(tinfo.tunnel_status)
{
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_CAN_TALK :
cinfo.status = RS_DISTANT_CHAT_STATUS_CAN_TALK; break;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN:
cinfo.status = RS_DISTANT_CHAT_STATUS_TUNNEL_DN; break;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED:
cinfo.status = RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED; break;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_CAN_TALK : cinfo.status = RS_DISTANT_CHAT_STATUS_CAN_TALK; break;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN: cinfo.status = RS_DISTANT_CHAT_STATUS_TUNNEL_DN; break;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED: cinfo.status = RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED; break;
case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_UNKNOWN:
default:
cinfo.status = RS_DISTANT_CHAT_STATUS_UNKNOWN; break;

230
libretroshare/src/deep_search/channelsindex.cpp Normal file
View File

@ -0,0 +1,230 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/channelsindex.hpp"
#include "deep_search/commonutils.hpp"
uint32_t DeepChannelsIndex::search(
const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results, uint32_t maxResults )
{
results.clear();
std::unique_ptr<Xapian::Database> dbPtr(
DeepSearch::openReadOnlyDatabase(dbPath()) );
if(!dbPtr) return 0;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
Xapian::QueryParser queryparser;
//queryparser.set_stemmer(Xapian::Stem("en"));
queryparser.set_stemming_strategy(queryparser.STEM_SOME);
// Start of prefix configuration.
//queryparser.add_prefix("title", "S");
//queryparser.add_prefix("description", "XD");
// End of prefix configuration.
// And parse the query.
Xapian::Query query = queryparser.parse_query(queryStr);
// Use an Enquire object on the database to run the query.
Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet mset = enquire.get_mset(
0, maxResults ? maxResults : db.get_doccount() );
for ( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
{
const Xapian::Document& doc = m.get_document();
DeepChannelsSearchResult s;
s.mUrl = doc.get_value(URL_VALUENO);
#if XAPIAN_AT_LEAST(1,3,5)
s.mSnippet = mset.snippet(doc.get_data());
#endif // XAPIAN_AT_LEAST(1,3,5)
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
}
void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan)
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(chan.mMeta.mGroupName, 1, "G");
termgenerator.index_text(
DeepSearch::timetToXapianDate(chan.mMeta.mPublishTs), 1, "D" );
termgenerator.index_text(chan.mDescription, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(chan.mMeta.mGroupName);
termgenerator.increase_termpos();
termgenerator.index_text(chan.mDescription);
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", chan.mMeta.mGroupId.toStdString());
const std::string idTerm("Q" + chanUrl.toString());
chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs));
chanUrl.setQueryKV("name", chan.mMeta.mGroupName);
if(!chan.mMeta.mAuthorId.isNull())
chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString());
if(chan.mMeta.mSignFlags)
chanUrl.setQueryKV( "signFlags",
std::to_string(chan.mMeta.mSignFlags) );
std::string rsLink(chanUrl.toString());
// store the RS link so we are able to retrive it on matching search
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
doc.set_data(chan.mMeta.mGroupName + "\n" + chan.mDescription);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer. "Q" prefix is a Xapian convention for unique id term.
doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
}
void DeepChannelsIndex::removeChannelFromIndex(RsGxsGroupId grpId)
{
// "Q" prefix is a Xapian convention for unique id term.
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", grpId.toStdString());
std::string idTerm("Q" + chanUrl.toString());
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
}
void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post)
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
termgenerator.index_text(
DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" );
// TODO: we should strip out HTML tags instead of skipping indexing
// Avoid indexing HTML
bool isPlainMsg =
post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>';
if(isPlainMsg)
termgenerator.index_text(post.mMsg, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName);
if(isPlainMsg)
{
termgenerator.increase_termpos();
termgenerator.index_text(post.mMsg);
}
for(const RsGxsFile& attachment : post.mFiles)
{
termgenerator.index_text(attachment.mName, 1, "F");
termgenerator.increase_termpos();
termgenerator.index_text(attachment.mName);
}
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", post.mMeta.mGroupId.toStdString())
.setQueryKV("msgid", post.mMeta.mMsgId.toStdString());
std::string idTerm("Q" + postUrl.toString());
postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs));
postUrl.setQueryKV("name", post.mMeta.mMsgName);
postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString());
std::string rsLink(postUrl.toString());
// store the RS link so we are able to retrive it on matching search
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
if(isPlainMsg)
doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg);
else doc.set_data(post.mMeta.mMsgName);
doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
}
void DeepChannelsIndex::removeChannelPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId )
{
RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", grpId.toStdString())
.setQueryKV("msgid", msgId.toStdString());
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postUrl.toString());
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
DeepSearch::openWritableDatabase(
dbPath(), Xapian::DB_CREATE_OR_OPEN ) );
if(!dbPtr) return;
Xapian::WritableDatabase& db(*dbPtr);
db.delete_document(idTerm);
}

77
libretroshare/src/deep_search/channelsindex.hpp Normal file
View File

@ -0,0 +1,77 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include <vector>
#include <xapian.h>
#include "util/rstime.h"
#include "retroshare/rsgxschannels.h"
#include "retroshare/rsinit.h"
#include "util/rsurl.h"
struct DeepChannelsSearchResult
{
std::string mUrl;
double mWeight;
std::string mSnippet;
};
struct DeepChannelsIndex
{
/**
* @brief Search indexed GXS groups and messages
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
static uint32_t search( const std::string& queryStr,
std::vector<DeepChannelsSearchResult>& results,
uint32_t maxResults = 100 );
static void indexChannelGroup(const RsGxsChannelGroup& chan);
static void removeChannelFromIndex(RsGxsGroupId grpId);
static void indexChannelPost(const RsGxsChannelPost& post);
static void removeChannelPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId );
static uint32_t indexFile(const std::string& path);
private:
enum : Xapian::valueno
{
/// Used to store retroshare url of indexed documents
URL_VALUENO,
/// @see Xapian::BAD_VALUENO
BAD_VALUENO = Xapian::BAD_VALUENO
};
static const std::string& dbPath()
{
static const std::string dbDir =
RsAccounts::AccountDirectory() + "/deep_channels_xapian_db";
return dbDir;
}
};

93
libretroshare/src/deep_search/commonutils.cpp Normal file
View File

@ -0,0 +1,93 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/commonutils.hpp"
#include "util/stacktrace.h"
#include "util/rsdebug.h"
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags, int blockSize )
{
try
{
std::unique_ptr<Xapian::WritableDatabase> dbPtr(
new Xapian::WritableDatabase(path, flags, blockSize) );
return dbPtr;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags )
{
try
{
std::unique_ptr<Xapian::Database> dbPtr(
new Xapian::Database(path, flags) );
return dbPtr;
}
catch(Xapian::DatabaseOpeningError e)
{
RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet." << std::endl;
}
catch(Xapian::DatabaseLockError)
{
RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock "
<< path << std::endl;
print_stacktrace();
}
catch(...)
{
RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted "
<< "deleting it might help without causing any harm: "
<< path << std::endl;
print_stacktrace();
}
return nullptr;
}
std::string timetToXapianDate(const rstime_t& time)
{
char date[] = "YYYYMMDD\0";
time_t tTime = static_cast<time_t>(time);
std::strftime(date, 9, "%Y%m%d", std::gmtime(&tTime));
return date;
}
}

45
libretroshare/src/deep_search/commonutils.hpp Normal file
View File

@ -0,0 +1,45 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include <xapian.h>
#include <memory>
#include "util/rstime.h"
#ifndef XAPIAN_AT_LEAST
#define XAPIAN_AT_LEAST(A,B,C) (XAPIAN_MAJOR_VERSION > (A) || \
(XAPIAN_MAJOR_VERSION == (A) && \
(XAPIAN_MINOR_VERSION > (B) || \
(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
#endif // ndef XAPIAN_AT_LEAST
namespace DeepSearch
{
std::unique_ptr<Xapian::WritableDatabase> openWritableDatabase(
const std::string& path, int flags = 0, int blockSize = 0 );
std::unique_ptr<Xapian::Database> openReadOnlyDatabase(
const std::string& path, int flags = 0 );
std::string timetToXapianDate(const rstime_t& time);
}

276
libretroshare/src/deep_search/deep_search.h
View File

@ -1,276 +0,0 @@
/*******************************************************************************
* libretroshare/src/crypto: crypto.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include "util/rstime.h"
#include <vector>
#include <xapian.h>
#include "retroshare/rsgxschannels.h"
#include "retroshare/rsinit.h"
#include "util/rsurl.h"
#ifndef XAPIAN_AT_LEAST
#define XAPIAN_AT_LEAST(A,B,C) (XAPIAN_MAJOR_VERSION > (A) || \
(XAPIAN_MAJOR_VERSION == (A) && \
(XAPIAN_MINOR_VERSION > (B) || \
(XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C)))))
#endif // ndef XAPIAN_AT_LEAST
struct DeepSearch
{
struct SearchResult
{
std::string mUrl;
std::string mSnippet;
};
/**
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
static uint32_t search( const std::string& queryStr,
std::vector<SearchResult>& results,
uint32_t maxResults = 100 )
{
results.clear();
Xapian::Database db;
// Open the database we're going to search.
try { db = Xapian::Database(dbPath()); }
catch(Xapian::DatabaseOpeningError e)
{
std::cerr << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< ", probably nothing has been indexed yet."<< std::endl;
return 0;
}
catch(Xapian::DatabaseError e)
{
std::cerr << __PRETTY_FUNCTION__ << " " << e.get_msg()
<< " this is fishy, maybe " << dbPath()
<< " has been corrupted (deleting it may help in that "
<< "case without loosing data)" << std::endl;
return 0;
}
// Set up a QueryParser with a stemmer and suitable prefixes.
Xapian::QueryParser queryparser;
//queryparser.set_stemmer(Xapian::Stem("en"));
queryparser.set_stemming_strategy(queryparser.STEM_SOME);
// Start of prefix configuration.
//queryparser.add_prefix("title", "S");
//queryparser.add_prefix("description", "XD");
// End of prefix configuration.
// And parse the query.
Xapian::Query query = queryparser.parse_query(queryStr);
// Use an Enquire object on the database to run the query.
Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet mset = enquire.get_mset(
0, maxResults ? maxResults : db.get_doccount() );
for ( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
{
const Xapian::Document& doc = m.get_document();
SearchResult s;
s.mUrl = doc.get_value(URL_VALUENO);
#if XAPIAN_AT_LEAST(1,3,5)
s.mSnippet = mset.snippet(doc.get_data());
#endif // XAPIAN_AT_LEAST(1,3,5)
results.push_back(s);
}
return results.size();
}
static void indexChannelGroup(const RsGxsChannelGroup& chan)
{
Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN);
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(chan.mMeta.mGroupName, 1, "G");
termgenerator.index_text(timetToXapianDate(chan.mMeta.mPublishTs), 1, "D");
termgenerator.index_text(chan.mDescription, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(chan.mMeta.mGroupName);
termgenerator.increase_termpos();
termgenerator.index_text(chan.mDescription);
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", chan.mMeta.mGroupId.toStdString());
const std::string idTerm("Q" + chanUrl.toString());
chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs));
chanUrl.setQueryKV("name", chan.mMeta.mGroupName);
if(!chan.mMeta.mAuthorId.isNull())
chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString());
if(chan.mMeta.mSignFlags)
chanUrl.setQueryKV( "signFlags",
std::to_string(chan.mMeta.mSignFlags) );
std::string rsLink(chanUrl.toString());
// store the RS link so we are able to retrive it on matching search
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
doc.set_data(chan.mMeta.mGroupName + "\n" + chan.mDescription);
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer. "Q" prefix is a Xapian convention for unique id term.
doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
}
static void removeChannelFromIndex(RsGxsGroupId grpId)
{
// "Q" prefix is a Xapian convention for unique id term.
RsUrl chanUrl; chanUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", grpId.toStdString());
std::string idTerm("Q" + chanUrl.toString());
Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN);
db.delete_document(idTerm);
}
static void indexChannelPost(const RsGxsChannelPost& post)
{
Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN);
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
// We make a document and tell the term generator to use this.
Xapian::Document doc;
termgenerator.set_document(doc);
// Index each field with a suitable prefix.
termgenerator.index_text(post.mMeta.mMsgName, 1, "S");
termgenerator.index_text(timetToXapianDate(post.mMeta.mPublishTs), 1, "D");
// Avoid indexing HTML
bool isPlainMsg = post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>';
if(isPlainMsg)
termgenerator.index_text(post.mMsg, 1, "XD");
// Index fields without prefixes for general search.
termgenerator.index_text(post.mMeta.mMsgName);
if(isPlainMsg)
{
termgenerator.increase_termpos();
termgenerator.index_text(post.mMsg);
}
for(const RsGxsFile& attachment : post.mFiles)
{
termgenerator.index_text(attachment.mName, 1, "F");
termgenerator.increase_termpos();
termgenerator.index_text(attachment.mName);
}
// We use the identifier to ensure each object ends up in the
// database only once no matter how many times we run the
// indexer.
RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", post.mMeta.mGroupId.toStdString())
.setQueryKV("msgid", post.mMeta.mMsgId.toStdString());
std::string idTerm("Q" + postUrl.toString());
postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs));
postUrl.setQueryKV("name", post.mMeta.mMsgName);
postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString());
std::string rsLink(postUrl.toString());
// store the RS link so we are able to retrive it on matching search
doc.add_value(URL_VALUENO, rsLink);
// Store some fields for display purposes.
if(isPlainMsg)
doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg);
else doc.set_data(post.mMeta.mMsgName);
doc.add_boolean_term(idTerm);
db.replace_document(idTerm, doc);
}
static void removeChannelPostFromIndex(
RsGxsGroupId grpId, RsGxsMessageId msgId )
{
RsUrl postUrl; postUrl
.setScheme("retroshare").setPath("/channel")
.setQueryKV("id", grpId.toStdString())
.setQueryKV("msgid", msgId.toStdString());
// "Q" prefix is a Xapian convention for unique id term.
std::string idTerm("Q" + postUrl.toString());
Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN);
db.delete_document(idTerm);
}
private:
enum : Xapian::valueno
{
/// Used to store retroshare url of indexed documents
URL_VALUENO,
/// @see Xapian::BAD_VALUENO
BAD_VALUENO = Xapian::BAD_VALUENO
};
static const std::string& dbPath()
{
static const std::string dbDir =
RsAccounts::AccountDirectory() + "/deep_search_xapian_db";
return dbDir;
}
static std::string timetToXapianDate(const rstime_t& time)
{
char date[] = "YYYYMMDD\0";
time_t tTime = static_cast<time_t>(time);
std::strftime(date, 9, "%Y%m%d", std::gmtime(&tTime));
return date;
}
};

156
libretroshare/src/deep_search/filesflacindexer.hpp Normal file
View File

@ -0,0 +1,156 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "util/rsdebug.h"
#include <xapian.h>
#include <string>
#include <FLAC++/metadata.h>
#include <cctype>
#include <memory>
struct RsDeepFlacFileIndexer
{
RsDeepFlacFileIndexer()
{
DeepFilesIndex::registerIndexer(31, indexFlacFile);
}
static uint32_t indexFlacFile(
const std::string& path, const std::string& /*name*/,
Xapian::TermGenerator& xTG, Xapian::Document& xDoc )
{
Dbg3() << __PRETTY_FUNCTION__ << " " << path << std::endl;
using FlacChain = FLAC::Metadata::Chain;
std::unique_ptr<FlacChain> flacChain(new FlacChain);
if(!flacChain->is_valid())
{
RsErr() << __PRETTY_FUNCTION__ << " Failed creating FLAC Chain 1"
<< std::endl;
return 1;
}
if(!flacChain->read(path.c_str(), false))
{
Dbg3() << __PRETTY_FUNCTION__ << " Failed to open the file as FLAC"
<< std::endl;
flacChain.reset(new FlacChain);
if(!flacChain->is_valid())
{
RsErr() << __PRETTY_FUNCTION__
<< " Failed creating FLAC Chain 2" << std::endl;
return 1;
}
if(!flacChain->read(path.c_str(), true))
{
Dbg3() << __PRETTY_FUNCTION__
<< " Failed to open the file as OggFLAC"
<< std::endl;
return 0;
}
}
unsigned validCommentsCnt = 0;
std::string docData = xDoc.get_data();
FLAC::Metadata::Iterator mdit;
mdit.init(*flacChain);
if(!mdit.is_valid()) return 1;
do
{
::FLAC__MetadataType mdt = mdit.get_block_type();
if (mdt != FLAC__METADATA_TYPE_VORBIS_COMMENT) continue;
Dbg2() << __PRETTY_FUNCTION__ << " Found Vorbis Comment Block"
<< std::endl;
std::unique_ptr<FLAC::Metadata::Prototype> proto(mdit.get_block());
if(!proto) continue;
const FLAC::Metadata::VorbisComment* vc =
dynamic_cast<FLAC::Metadata::VorbisComment*>(proto.get());
if(!vc || !vc->is_valid()) continue;
unsigned numComments = vc->get_num_comments();
for(unsigned i = 0; i < numComments; ++i)
{
FLAC::Metadata::VorbisComment::Entry entry =
vc->get_comment(i);
if(!entry.is_valid()) continue;
std::string tagName( entry.get_field_name(),
entry.get_field_name_length() );
/* Vorbis tags should be uppercases but not all the softwares
* enforce it */
for (auto& c: tagName) c = static_cast<char>(toupper(c));
std::string tagValue( entry.get_field_value(),
entry.get_field_value_length() );
if(tagValue.empty()) continue;
if(tagName == "ARTIST")
xTG.index_text(tagValue, 1, "A");
else if (tagName == "DESCRIPTION")
xTG.index_text(tagValue, 1, "XD");
else if (tagName == "TITLE")
xTG.index_text(tagValue, 1, "S");
else if(tagName.find("COVERART") != tagName.npos)
continue; // Avoid polluting the index with binary data
else if (tagName.find("METADATA_BLOCK_PICTURE") != tagName.npos)
continue; // Avoid polluting the index with binary data
// Index fields without prefixes for general search.
xTG.increase_termpos();
std::string fullComment(tagName + "=" + tagValue);
xTG.index_text(fullComment);
docData += fullComment + "\n";
Dbg2() << __PRETTY_FUNCTION__ << " Indexed " << fullComment
<< std::endl;
++validCommentsCnt;
}
}
while(mdit.next());
if(validCommentsCnt > 0)
{
Dbg1() << __PRETTY_FUNCTION__ << " Successfully indexed: " << path
<< std::endl;
xDoc.set_data(docData);
return 99;
}
/* Altought the file appears to be a valid FLAC, no vorbis comment has
* been found so return less then 50 maybe it has tagged only with ID3
* tags ? */
return 30;
}
RS_SET_CONTEXT_DEBUG_LEVEL(3)
};

171
libretroshare/src/deep_search/filesindex.cpp Normal file
View File

@ -0,0 +1,171 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "deep_search/commonutils.hpp"
#include "util/rsdebug.h"
#include "retroshare/rsinit.h"
#include "retroshare/rsversion.h"
#include <utility>
/*static*/ std::multimap<int, DeepFilesIndex::IndexerFunType>
DeepFilesIndex::indexersRegister = {};
bool DeepFilesIndex::indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash )
{
auto dbPtr = DeepSearch::openWritableDatabase(
mDbPath, Xapian::DB_CREATE_OR_OPEN );
if(!dbPtr) return false;
Xapian::WritableDatabase& db(*dbPtr);
const std::string hashString = hash.toStdString();
const std::string idTerm("Q" + hashString);
Xapian::Document oldDoc;
Xapian::PostingIterator pIt = db.postlist_begin(idTerm);
if( pIt != db.postlist_end(idTerm) )
{
oldDoc = db.get_document(*pIt);
if( oldDoc.get_value(INDEXER_VERSION_VALUENO) ==
RS_HUMAN_READABLE_VERSION &&
std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) ==
indexersRegister.size() )
{
/* Looks like this file has already been indexed by this RetroShare
* exact version, so we can skip it. If the version was different it
* made sense to reindex it as better indexers might be available
* since last time it was indexed */
Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: "
<< hash << " " << name << std::endl;
return true;
}
}
Xapian::Document doc;
// Set up a TermGenerator that we'll use in indexing.
Xapian::TermGenerator termgenerator;
//termgenerator.set_stemmer(Xapian::Stem("en"));
termgenerator.set_document(doc);
for(auto& indexerPair : indexersRegister)
if(indexerPair.second(path, name, termgenerator, doc) > 50)
break;
doc.add_boolean_term(idTerm);
termgenerator.index_text(name, 1, "N");
termgenerator.index_text(name);
doc.add_value(FILE_HASH_VALUENO, hashString);
doc.add_value(INDEXER_VERSION_VALUENO, RS_HUMAN_READABLE_VERSION);
doc.add_value(
INDEXERS_COUNT_VALUENO,
std::to_string(indexersRegister.size()) );
db.replace_document(idTerm, doc);
return true;
}
bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash)
{
Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: "
<< hash << std::endl;
std::unique_ptr<Xapian::WritableDatabase> db =
DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN);
if(!db) return false;
db->delete_document("Q" + hash.toStdString());
return true;
}
/*static*/ std::string DeepFilesIndex::dbDefaultPath()
{ return RsAccounts::AccountDirectory() + "/deep_files_index_xapian_db"; }
/*static*/ bool DeepFilesIndex::registerIndexer(
int order, const DeepFilesIndex::IndexerFunType& indexerFun )
{
Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl;
indexersRegister.insert(std::make_pair(order, indexerFun));
return true;
}
uint32_t DeepFilesIndex::search(
const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results, uint32_t maxResults )
{
results.clear();
auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath);
if(!dbPtr) return 0;
Xapian::Database& db(*dbPtr);
// Set up a QueryParser with a stemmer and suitable prefixes.
Xapian::QueryParser queryparser;
//queryparser.set_stemmer(Xapian::Stem("en"));
queryparser.set_stemming_strategy(queryparser.STEM_SOME);
// Start of prefix configuration.
//queryparser.add_prefix("title", "S");
//queryparser.add_prefix("description", "XD");
// End of prefix configuration.
// And parse the query.
Xapian::Query query = queryparser.parse_query(queryStr);
// Use an Enquire object on the database to run the query.
Xapian::Enquire enquire(db);
enquire.set_query(query);
Xapian::MSet mset = enquire.get_mset(
0, maxResults ? maxResults : db.get_doccount() );
for ( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m )
{
const Xapian::Document& doc = m.get_document();
DeepFilesSearchResult s;
s.mFileHash = RsFileHash(doc.get_value(FILE_HASH_VALUENO));
s.mWeight = m.get_weight();
#if XAPIAN_AT_LEAST(1,3,5)
s.mSnippet = mset.snippet(doc.get_data());
#endif // XAPIAN_AT_LEAST(1,3,5)
results.push_back(s);
}
return static_cast<uint32_t>(results.size());
}
#ifdef RS_DEEP_FILES_INDEX_OGG
# include "deep_search/filesoggindexer.hpp"
static RsDeepOggFileIndexer oggFileIndexer;
#endif // def RS_DEEP_FILES_INDEX_OGG
#ifdef RS_DEEP_FILES_INDEX_FLAC
# include "deep_search/filesflacindexer.hpp"
static RsDeepFlacFileIndexer flacFileIndexer;
#endif // def RS_DEEP_FILES_INDEX_FLAC
#ifdef RS_DEEP_FILES_INDEX_TAGLIB
# include "deep_search/filestaglibindexer.hpp"
static RsDeepTaglibFileIndexer taglibFileIndexer;
#endif // def RS_DEEP_FILES_INDEX_TAGLIB

103
libretroshare/src/deep_search/filesindex.hpp Normal file
View File

@ -0,0 +1,103 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
#include "retroshare/rstypes.h"
#include "util/rsdebug.h"
#include <string>
#include <cstdint>
#include <vector>
#include <xapian.h>
#include <map>
#include <functional>
struct DeepFilesSearchResult
{
DeepFilesSearchResult() : mWeight(0) {}
RsFileHash mFileHash;
double mWeight;
std::string mSnippet;
};
class DeepFilesIndex
{
public:
DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {}
/**
* @brief Search indexed files
* @param[in] maxResults maximum number of acceptable search results, 0 for
* no limits
* @return search results count
*/
uint32_t search( const std::string& queryStr,
std::vector<DeepFilesSearchResult>& results,
uint32_t maxResults = 100 );
/**
* @return false if file could not be indexed because of error or
* unsupported type, true otherwise.
*/
bool indexFile(
const std::string& path, const std::string& name,
const RsFileHash& hash );
/**
* @brief Remove file entry from database
* @return false on error, true otherwise.
*/
bool removeFileFromIndex(const RsFileHash& hash);
static std::string dbDefaultPath();
using IndexerFunType = std::function<
uint32_t( const std::string& path, const std::string& name,
Xapian::TermGenerator& xTG, Xapian::Document& xDoc ) >;
static bool registerIndexer(
int order, const IndexerFunType& indexerFun );
private:
enum : Xapian::valueno
{
/// Used to store RsFileHash of indexed documents
FILE_HASH_VALUENO,
/** Used to check if some file need reindex because was indexed with an
* older version of the indexer */
INDEXER_VERSION_VALUENO,
/** Used to check if some file need reindex because was indexed with an
* older version of the indexer */
INDEXERS_COUNT_VALUENO,
/// @see Xapian::BAD_VALUENO
BAD_VALUENO = Xapian::BAD_VALUENO
};
const std::string mDbPath;
/** Storage for indexers function by order */
static std::multimap<int, IndexerFunType> indexersRegister;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

97
libretroshare/src/deep_search/filesoggindexer.hpp Normal file
View File

@ -0,0 +1,97 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "util/rsdebug.h"
#include <xapian.h>
#include <string>
#include <vorbis/vorbisfile.h>
#include <cctype>
struct RsDeepOggFileIndexer
{
RsDeepOggFileIndexer()
{
DeepFilesIndex::registerIndexer(30, indexOggFile);
}
static uint32_t indexOggFile(
const std::string& path, const std::string& /*name*/,
Xapian::TermGenerator& xTG, Xapian::Document& xDoc )
{
Dbg3() << __PRETTY_FUNCTION__ << " " << path << std::endl;
OggVorbis_File vf;
int ret = ov_fopen(path.c_str(), &vf);
if(ret == 0 && vf.vc)
{
vorbis_comment& vc = *vf.vc;
std::string docData = xDoc.get_data();
for (int i = 0; i < vc.comments; ++i)
{
using szt = std::string::size_type;
std::string userComment(
vc.user_comments[i],
static_cast<szt>(vc.comment_lengths[i]) );
if(userComment.empty()) continue;
szt equalPos = userComment.find('=');
if(equalPos == std::string::npos) continue;
std::string tagName = userComment.substr(0, equalPos);
if(tagName.empty()) continue;
std::string tagValue = userComment.substr(equalPos + 1);
if(tagValue.empty()) continue;
/* Ogg tags should be uppercases but not all the softwares
* enforce it */
for (auto& c: tagName) c = static_cast<char>(toupper(c));
if(tagName == "ARTIST")
xTG.index_text(tagValue, 1, "A");
else if (tagName == "DESCRIPTION")
xTG.index_text(tagValue, 1, "XD");
else if (tagName == "TITLE")
xTG.index_text(tagValue, 1, "S");
else if(tagName.find("COVERART") != tagName.npos)
continue; // Avoid polluting the index with binary data
else if (tagName.find("METADATA_BLOCK_PICTURE") != tagName.npos)
continue; // Avoid polluting the index with binary data
// Index fields without prefixes for general search.
xTG.increase_termpos();
xTG.index_text(userComment);
docData += userComment + "\n";
}
xDoc.set_data(docData);
ov_clear(&vf);
return 99;
}
return 0;
}
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

103
libretroshare/src/deep_search/filestaglibindexer.hpp Normal file
View File

@ -0,0 +1,103 @@
/*******************************************************************************
* RetroShare full text indexing and search implementation based on Xapian *
* *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Affero General Public License version 3 as *
* published by the Free Software Foundation. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Affero General Public License for more details. *
* *
* You should have received a copy of the GNU Affero General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "deep_search/filesindex.hpp"
#include "util/rsdebug.h"
#include <xapian.h>
#include <string>
#include <memory>
#include <taglib/tag.h>
#include <taglib/fileref.h>
#include <taglib/tpropertymap.h>
struct RsDeepTaglibFileIndexer
{
RsDeepTaglibFileIndexer()
{
DeepFilesIndex::registerIndexer(40, indexFile);
}
static uint32_t indexFile(
const std::string& path, const std::string& /*name*/,
Xapian::TermGenerator& xTG, Xapian::Document& xDoc )
{
Dbg4() << __PRETTY_FUNCTION__ << " " << path << std::endl;
TagLib::FileRef tFile(path.c_str());
if(tFile.isNull()) return 0;
const TagLib::Tag* tag = tFile.tag();
if(!tag) return 0;
TagLib::PropertyMap tMap = tag->properties();
unsigned validCommentsCnt = 0;
std::string docData = xDoc.get_data();
for( TagLib::PropertyMap::ConstIterator mIt = tMap.begin();
mIt != tMap.end(); ++mIt )
{
if(mIt->first.isNull() || mIt->first.isEmpty()) continue;
std::string tagName(mIt->first.upper().to8Bit());
if(mIt->second.isEmpty()) continue;
std::string tagValue(mIt->second.toString(", ").to8Bit(true));
if(tagValue.empty()) continue;
if(tagName == "ARTIST")
xTG.index_text(tagValue, 1, "A");
else if (tagName == "DESCRIPTION")
xTG.index_text(tagValue, 1, "XD");
else if (tagName == "TITLE")
xTG.index_text(tagValue, 1, "S");
else if(tagName.find("COVERART") != tagName.npos)
continue; // Avoid polluting the index with binary data
else if (tagName.find("METADATA_BLOCK_PICTURE") != tagName.npos)
continue; // Avoid polluting the index with binary data
// Index fields without prefixes for general search.
xTG.increase_termpos();
std::string fullComment(tagName + "=" + tagValue);
xTG.index_text(fullComment);
docData += fullComment + "\n";
Dbg2() << __PRETTY_FUNCTION__ << " Indexed " << tagName << "=\""
<< tagValue << '"' << std::endl;
++validCommentsCnt;
}
if(validCommentsCnt > 0)
{
Dbg1() << __PRETTY_FUNCTION__ << " Successfully indexed: " << path
<< std::endl;
xDoc.set_data(docData);
return 99;
}
/* Altought the file appears to be supported by taglib, no comments has
* been found so return less then 50 maybe another indexer is capable of
* extracting information */
return 30;
}
RS_SET_CONTEXT_DEBUG_LEVEL(3)
};

15
libretroshare/src/file_sharing/dir_hierarchy.cc
View File

@ -21,15 +21,19 @@
******************************************************************************/
#include <sstream>
#include <algorithm>
#include "util/rstime.h"
#include "util/rsdir.h"
#include "util/rsprint.h"
#include "retroshare/rsexpr.h"
#include "dir_hierarchy.h"
#include "filelist_io.h"
#include "file_sharing_defaults.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
#endif // def RS_DEEP_FILES_INDEX
//#define DEBUG_DIRECTORY_STORAGE 1
typedef FileListIO::read_error read_error;
@ -391,6 +395,11 @@ void InternalFileHierarchyStorage::deleteFileNode(uint32_t index)
{
FileEntry& fe(*static_cast<FileEntry*>(mNodes[index])) ;
#ifdef RS_DEEP_FILES_INDEX
DeepFilesIndex tfi(DeepFilesIndex::dbDefaultPath());
tfi.removeFileFromIndex(fe.file_hash);
#endif
if(mTotalSize >= fe.file_size)
mTotalSize -= fe.file_size ;
@ -753,7 +762,9 @@ int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression
return 0;
}
int InternalFileHierarchyStorage::searchTerms(const std::list<std::string>& terms, std::list<DirectoryStorage::EntryIndex> &results) const
int InternalFileHierarchyStorage::searchTerms(
const std::list<std::string>& terms,
std::list<DirectoryStorage::EntryIndex>& results ) const
{
// most entries are likely to be files, so we could do a linear search over the entries tab.
// instead we go through the table of hashes.

44
libretroshare/src/file_sharing/directory_storage.cc
View File

@ -20,6 +20,7 @@
* *
******************************************************************************/
#include <set>
#include "util/rstime.h"
#include "serialiser/rstlvbinary.h"
#include "retroshare/rspeers.h"
@ -30,6 +31,10 @@
#include "dir_hierarchy.h"
#include "filelist_io.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
#endif // def RS_DEEP_FILES_INDEX
//#define DEBUG_REMOTE_DIRECTORY_STORAGE 1
/******************************************************************************************************************/
@ -180,7 +185,9 @@ void DirectoryStorage::print()
mFileHierarchy->print();
}
int DirectoryStorage::searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const
int DirectoryStorage::searchTerms(
const std::list<std::string>& terms,
std::list<EntryIndex>& results ) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchTerms(terms,results);
@ -501,18 +508,39 @@ void LocalDirectoryStorage::updateTimeStamps()
#endif
}
}
bool LocalDirectoryStorage::updateHash(const EntryIndex& index, const RsFileHash& hash, bool update_internal_hierarchy)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mEncryptedHashes[makeEncryptedHash(hash)] = hash ;
mChanged = true ;
bool LocalDirectoryStorage::updateHash(
const EntryIndex& index, const RsFileHash& hash,
bool update_internal_hierarchy )
{
bool ret = false;
{
RS_STACK_MUTEX(mDirStorageMtx);
mEncryptedHashes[makeEncryptedHash(hash)] = hash ;
mChanged = true ;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Updating index of hash " << hash << " update_internal=" << update_internal_hierarchy << std::endl;
std::cerr << "Updating index of hash " << hash << " update_internal="
<< update_internal_hierarchy << std::endl;
#endif
return (!update_internal_hierarchy)|| mFileHierarchy->updateHash(index,hash);
ret = (!update_internal_hierarchy) ||
mFileHierarchy->updateHash(index,hash);
} // RS_STACK_MUTEX(mDirStorageMtx);
#ifdef RS_DEEP_FILES_INDEX
FileInfo fInfo;
if( ret && getFileInfo(index, fInfo) &&
fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH )
{
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash);
}
#endif // def RS_DEEP_FILES_INDEX
return ret;
}
std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const
{

229
libretroshare/src/ft/ftserver.cc
View File

@ -54,6 +54,10 @@
#include <iostream>
#include "util/rstime.h"
#ifdef RS_DEEP_FILES_INDEX
# include "deep_search/filesindex.hpp"
#endif // def RS_DEEP_FILES_INDEX
/***
* #define SERVER_DEBUG 1
* #define SERVER_DEBUG_CACHE 1
@ -65,9 +69,26 @@
static const rstime_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priority tasks handling every 5 seconds
static const rstime_t FILE_TRANSFER_MAX_DELAY_BEFORE_DROP_USAGE_RECORD = 10 ; // keep usage records for 10 secs at most.
#ifdef RS_DEEP_FILES_INDEX
TurtleFileInfoV2::TurtleFileInfoV2(const DeepFilesSearchResult& dRes) :
fHash(dRes.mFileHash), fWeight(static_cast<float>(dRes.mWeight)),
fSnippet(dRes.mSnippet)
{
FileInfo fInfo;
rsFiles->FileDetails(fHash, RS_FILE_HINTS_LOCAL, fInfo);
fSize = fInfo.size;
fName = fInfo.fname;
}
#endif // def RS_DEEP_FILES_INDEX
TurtleFileInfoV2::~TurtleFileInfoV2() = default;
/* Setup */
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc)
: p3Service(),RsServiceSerializer(RS_SERVICE_TYPE_TURTLE), // should be FT, but this is for backward compatibility
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc):
p3Service(),
// should be FT, but this is for backward compatibility
RsServiceSerializer(RS_SERVICE_TYPE_TURTLE),
mPeerMgr(pm), mServiceCtrl(sc),
mFileDatabase(NULL),
mFtController(NULL), mFtExtra(NULL),
@ -500,15 +521,24 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi
return false;
}
RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const
RsItem *ftServer::create_item(uint16_t service, uint8_t item_type) const
{
#ifdef SERVER_DEBUG
FTSERVER_DEBUG() << "p3turtle: deserialising packet: " << std::endl ;
#endif
if (RS_SERVICE_TYPE_TURTLE != service)
RsServiceType serviceType = static_cast<RsServiceType>(service);
switch (serviceType)
{
FTSERVER_ERROR() << " Wrong type !!" << std::endl ;
return NULL; /* wrong type */
/* This one is here for retro-compatibility as turtle routing and file
* trasfer services were just one service before turle service was
* generalized */
case RsServiceType::TURTLE: break;
case RsServiceType::FILE_TRANSFER: break;
default:
RsErr() << __PRETTY_FUNCTION__ << " Wrong service type: " << service
<< std::endl;
return nullptr;
}
try
@ -521,16 +551,19 @@ RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const
case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem();
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST : return new RsTurtleChunkCrcRequestItem();
case RS_TURTLE_SUBTYPE_CHUNK_CRC : return new RsTurtleChunkCrcItem();
case static_cast<uint8_t>(RsFileItemType::FILE_SEARCH_REQUEST):
return new RsFileSearchRequestItem();
case static_cast<uint8_t>(RsFileItemType::FILE_SEARCH_RESULT):
return new RsFileSearchResultItem();
default:
return NULL ;
return nullptr;
}
}
catch(std::exception& e)
{
FTSERVER_ERROR() << "(EE) deserialisation error in " << __PRETTY_FUNCTION__ << ": " << e.what() << std::endl;
return NULL ;
return nullptr;
}
}
@ -1837,7 +1870,12 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item)
if(cbpt != mSearchCallbacksMap.end())
{
hasCallback = true;
cbpt->second.first(item->result);
std::vector<TurtleFileInfoV2> cRes;
for( const auto& tfiold : item->result)
cRes.push_back(tfiold);
cbpt->second.first(cRes);
}
} // end RS_STACK_MUTEX(mSearchCallbacksMapMutex);
@ -1845,6 +1883,99 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item)
RsServer::notify()->notifyTurtleSearchResult(item->PeerId(),item->request_id, item->result );
}
bool ftServer::receiveSearchRequest(
unsigned char* searchRequestData, uint32_t searchRequestDataLen,
unsigned char*& searchResultData, uint32_t& searchResultDataLen,
uint32_t& maxAllowsHits )
{
#ifdef RS_DEEP_FILES_INDEX
std::unique_ptr<RsItem> recvItem(
RsServiceSerializer::deserialise(
searchRequestData, &searchRequestDataLen ) );
if(!recvItem)
{
RsWarn() << __PRETTY_FUNCTION__ << " Search request deserialization "
<< "failed" << std::endl;
return false;
}
std::unique_ptr<RsFileSearchRequestItem> sReqItPtr(
dynamic_cast<RsFileSearchRequestItem*>(recvItem.get()) );
if(!sReqItPtr)
{
RsWarn() << __PRETTY_FUNCTION__ << " Received an invalid search request"
<< " " << *recvItem << std::endl;
return false;
}
recvItem.release();
RsFileSearchRequestItem& searchReq(*sReqItPtr);
std::vector<DeepFilesSearchResult> dRes;
DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath());
if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0)
{
RsFileSearchResultItem resIt;
for(const auto& dMatch : dRes)
resIt.mResults.push_back(TurtleFileInfoV2(dMatch));
searchResultDataLen = RsServiceSerializer::size(&resIt);
searchResultData = static_cast<uint8_t*>(malloc(searchResultDataLen));
return RsServiceSerializer::serialise(
&resIt, searchResultData, &searchResultDataLen );
}
#endif // def RS_DEEP_FILES_INDEX
searchResultData = nullptr;
searchResultDataLen = 0;
return false;
}
void ftServer::receiveSearchResult(
TurtleSearchRequestId requestId, unsigned char* searchResultData,
uint32_t searchResultDataLen )
{
if(!searchResultData || !searchResultDataLen)
{
RsWarn() << __PRETTY_FUNCTION__ << " got null paramethers "
<< "searchResultData: " << static_cast<void*>(searchResultData)
<< " searchResultDataLen: " << searchResultDataLen
<< " seems someone else in the network have a buggy RetroShare"
<< " implementation" << std::endl;
return;
}
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
auto cbpt = mSearchCallbacksMap.find(requestId);
if(cbpt != mSearchCallbacksMap.end())
{
RsItem* recvItem = RsServiceSerializer::deserialise(
searchResultData, &searchResultDataLen );
if(!recvItem)
{
RsWarn() << __PRETTY_FUNCTION__ << " Search result deserialization "
<< "failed" << std::endl;
return;
}
std::unique_ptr<RsFileSearchResultItem> resItPtr(
dynamic_cast<RsFileSearchResultItem*>(recvItem) );
if(!resItPtr)
{
RsWarn() << __PRETTY_FUNCTION__ << " Received invalid search result"
<< std::endl;
delete recvItem;
return;
}
cbpt->second.first(resItPtr->mResults);
}
}
/***************************** CONFIG ****************************/
bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
@ -1857,27 +1988,74 @@ bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
return true;
}
#ifdef RS_DEEP_FILES_INDEX
static std::vector<std::string> xapianQueryKeywords =
{
" AND ", " OR ", " NOT ", " XOR ", " +", " -", " ( ", " ) ", " NEAR ",
" ADJ ", " \"", "\" "
};
#endif
bool ftServer::turtleSearchRequest(
const std::string& matchString,
const std::function<void (const std::list<TurtleFileInfo>& results)>& multiCallback,
const std::function<void (const std::vector<TurtleFileInfoV2>& results)>& multiCallback,
rstime_t maxWait )
{
if(matchString.empty())
{
std::cerr << __PRETTY_FUNCTION__ << " match string can't be empty!"
<< std::endl;
RsWarn() << __PRETTY_FUNCTION__ << " match string can't be empty!"
<< std::endl;
return false;
}
TurtleRequestId sId = turtleSearch(matchString);
#ifdef RS_DEEP_FILES_INDEX
RsFileSearchRequestItem sItem;
sItem.queryString = matchString;
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
mSearchCallbacksMap.emplace(
sId,
std::make_pair(
multiCallback,
std::chrono::system_clock::now() +
std::chrono::seconds(maxWait) ) );
uint32_t iSize = RsServiceSerializer::size(&sItem);
uint8_t* iBuf = static_cast<uint8_t*>(malloc(iSize));
RsServiceSerializer::serialise(&sItem, iBuf, &iSize);
Dbg3() << __PRETTY_FUNCTION__ << " sending search request:" << sItem
<< std::endl;
TurtleRequestId xsId = mTurtleRouter->turtleSearch(iBuf, iSize, this);
{ RS_STACK_MUTEX(mSearchCallbacksMapMutex);
mSearchCallbacksMap.emplace(
xsId,
std::make_pair(
multiCallback,
std::chrono::system_clock::now() +
std::chrono::seconds(maxWait) ) );
} // RS_STACK_MUTEX(mSearchCallbacksMapMutex);
/* Trick to keep receiving more or less usable results from old peers */
std::string strippedQuery = matchString;
for(const std::string& xKeyword : xapianQueryKeywords)
{
std::string::size_type pos = std::string::npos;
while( (pos = strippedQuery.find(xKeyword)) != std::string::npos )
strippedQuery.replace(pos, xKeyword.length(), " ");
}
Dbg3() << __PRETTY_FUNCTION__ << " sending stripped query for "
<< "retro-compatibility: " << strippedQuery << std::endl;
TurtleRequestId sId = mTurtleRouter->turtleSearch(strippedQuery);
#else // def RS_DEEP_FILES_INDEX
TurtleRequestId sId = mTurtleRouter->turtleSearch(matchString);
#endif // def RS_DEEP_FILES_INDEX
{
RS_STACK_MUTEX(mSearchCallbacksMapMutex);
mSearchCallbacksMap.emplace(
sId,
std::make_pair(
multiCallback,
std::chrono::system_clock::now() +
std::chrono::seconds(maxWait) ) );
}
return true;
}
@ -1913,4 +2091,13 @@ bool ftServer::isHashBanned(const RsFileHash& hash)
return mFileDatabase->isFileBanned(hash);
}
RsFileItem::~RsFileItem() = default;
RsFileItem::RsFileItem(RsFileItemType subtype) :
RsItem( RS_PKT_VERSION_SERVICE,
static_cast<uint16_t>(RsServiceType::FILE_TRANSFER),
static_cast<uint8_t>(subtype) ) {}
void RsFileSearchRequestItem::clear() { queryString.clear(); }
void RsFileSearchResultItem::clear() { mResults.clear(); }

73
libretroshare/src/ft/ftserver.h
View File

@ -46,7 +46,8 @@
#include "turtle/turtleclientservice.h"
#include "services/p3service.h"
#include "retroshare/rsfiles.h"
#include "rsitems/rsitem.h"
#include "serialiser/rsserial.h"
#include "pqi/pqi.h"
#include "pqi/p3cfgmgr.h"
@ -67,7 +68,53 @@ class p3PeerMgr;
class p3ServiceControl;
class p3FileDatabase;
class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService, public RsServiceSerializer
enum class RsFileItemType : uint8_t
{
NONE = 0x00, /// Only to detect ununitialized
FILE_SEARCH_REQUEST = 0x57,
FILE_SEARCH_RESULT = 0x58
};
struct RsFileItem : RsItem
{
~RsFileItem() override;
protected:
RsFileItem(RsFileItemType subtype);
};
struct RsFileSearchRequestItem : RsFileItem
{
RsFileSearchRequestItem() : RsFileItem(RsFileItemType::FILE_SEARCH_REQUEST)
{ setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_REQUEST); }
std::string queryString;
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{ RS_SERIAL_PROCESS(queryString); }
void clear() override;
};
struct RsFileSearchResultItem : RsFileItem
{
RsFileSearchResultItem() : RsFileItem(RsFileItemType::FILE_SEARCH_RESULT)
{ setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT); }
std::vector<TurtleFileInfoV2> mResults;
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{ RS_SERIAL_PROCESS(mResults); }
void clear() override;
};
class ftServer :
public p3Service, public RsFiles, public ftDataSend,
public RsTurtleClientService, public RsServiceSerializer
{
public:
@ -98,7 +145,21 @@ public:
uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ;
virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
virtual void ftReceiveSearchResult(RsTurtleFTSearchResultItem *item); // We dont use TurtleClientService::receiveSearchResult() because of backward compatibility.
/// We keep this for retro-compatibility @see RsTurtleClientService
virtual void ftReceiveSearchResult(RsTurtleFTSearchResultItem *item);
/// @see RsTurtleClientService
bool receiveSearchRequest(
unsigned char* searchRequestData, uint32_t searchRequestDataLen,
unsigned char*& search_result_data, uint32_t& searchResultDataLen,
uint32_t& maxAllowsHits ) override;
/// @see RsTurtleClientService
void receiveSearchResult(
TurtleSearchRequestId requestId, unsigned char* searchResultData,
uint32_t searchResultDataLen ) override;
virtual RsItem *create_item(uint16_t service,uint8_t item_type) const ;
virtual RsServiceSerializer *serializer() { return this ; }
@ -148,7 +209,7 @@ public:
/// @see RsFiles
virtual bool turtleSearchRequest(
const std::string& matchString,
const std::function<void (const std::list<TurtleFileInfo>& results)>& multiCallback,
const std::function<void (const std::vector<TurtleFileInfoV2>& results)>& multiCallback,
rstime_t maxWait = 300 );
virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
@ -337,13 +398,15 @@ private:
std::map<
TurtleRequestId,
std::pair<
std::function<void (const std::list<TurtleFileInfo>& results)>,
std::function<void (const std::vector<TurtleFileInfoV2>& results)>,
std::chrono::system_clock::time_point >
> mSearchCallbacksMap;
RsMutex mSearchCallbacksMapMutex;
/// Cleanup mSearchCallbacksMap
void cleanTimedOutSearches();
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

2
libretroshare/src/gxs/rsgixs.h
View File

@ -165,6 +165,8 @@ public:
virtual bool getKey(const RsGxsId &id, RsTlvPublicRSAKey& key) = 0;
virtual bool getPrivateKey(const RsGxsId &id, RsTlvPrivateRSAKey& key) = 0; // For signing outgoing messages.
virtual bool getIdDetails(const RsGxsId& id, RsIdentityDetails& details) = 0 ; // Proxy function so that we get p3Identity info from Gxs
virtual ~RsGixs();
};
class GixsReputation

24
libretroshare/src/gxs/rsgxsnetservice.cc
View File

@ -257,8 +257,8 @@
#include "util/rsmemory.h"
#include "util/stacktrace.h"
#ifdef RS_DEEP_SEARCH
# include "deep_search/deep_search.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
#endif
/***
@ -5148,13 +5148,13 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
}
#ifndef RS_DEEP_SEARCH
#ifndef RS_DEEP_CHANNEL_INDEX
static bool termSearch(const std::string& src, const std::string& substring)
{
/* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
}
#endif // ndef RS_DEEP_SEARCH
#endif // ndef RS_DEEP_CHANNEL_INDEX
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos)
{
@ -5209,11 +5209,11 @@ void RsGxsNetService::receiveTurtleSearchResults(
for (const RsGxsGroupSummary& gps : group_infos)
{
#ifndef RS_DEEP_SEARCH
#ifndef RS_DEEP_CHANNEL_INDEX
/* Only keep groups that are not locally known, and groups that are
* not already in the mDistantSearchResults structure. */
if(grpMeta[gps.mGroupId]) continue;
#else // ndef RS_DEEP_SEARCH
#else // ndef RS_DEEP_CHANNEL_INDEX
/* When deep search is enabled search results may bring more info
* then we already have also about post that are indexed by xapian,
* so we don't apply this filter in this case. */
@ -5302,9 +5302,9 @@ bool RsGxsNetService::search( const std::string& substring,
{
group_infos.clear();
#ifdef RS_DEEP_SEARCH
std::vector<DeepSearch::SearchResult> results;
DeepSearch::search(substring, results);
#ifdef RS_DEEP_CHANNEL_INDEX
std::vector<DeepChannelsSearchResult> results;
DeepChannelsIndex::search(substring, results);
for(auto dsr : results)
{
@ -5324,7 +5324,7 @@ bool RsGxsNetService::search( const std::string& substring,
if((rit = uQ.find("name")) != uQ.end())
s.mGroupName = rit->second;
if((rit = uQ.find("signFlags")) != uQ.end())
s.mSignFlags = std::stoul(rit->second);
s.mSignFlags = static_cast<uint32_t>(std::stoul(rit->second));
if((rit = uQ.find("publishTs")) != uQ.end())
s.mPublishTs = static_cast<rstime_t>(std::stoll(rit->second));
if((rit = uQ.find("authorId")) != uQ.end())
@ -5340,7 +5340,7 @@ bool RsGxsNetService::search( const std::string& substring,
}
}
}
#else // RS_DEEP_SEARCH
#else // RS_DEEP_CHANNEL_INDEX
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -5366,7 +5366,7 @@ bool RsGxsNetService::search( const std::string& substring,
group_infos.push_back(s);
}
#endif // RS_DEEP_SEARCH
#endif // RS_DEEP_CHANNEL_INDEX
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;

42
libretroshare/src/gxs/rsgxsutil.cc
View File

@ -29,8 +29,8 @@
#include "pqi/pqihash.h"
#include "gxs/rsgixs.h"
#ifdef RS_DEEP_SEARCH
# include "deep_search/deep_search.h"
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
# include "services/p3gxschannels.h"
# include "rsitems/rsgxschannelitems.h"
#endif
@ -148,12 +148,12 @@ bool RsGxsMessageCleanUp::clean()
RsGxsIntegrityCheck::RsGxsIntegrityCheck(
RsGeneralDataService* const dataService, RsGenExchange* genex,
RsSerialType&
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
serializer
#endif
, RsGixs* gixs )
: mDs(dataService), mGenExchangeClient(genex),
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
mSerializer(serializer),
#endif
mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {}
@ -168,7 +168,7 @@ void RsGxsIntegrityCheck::run()
bool RsGxsIntegrityCheck::check()
{
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS;
std::set<RsGxsGroupId> indexedGroups;
#endif
@ -221,7 +221,7 @@ bool RsGxsIntegrityCheck::check()
}
else msgIds.erase(msgIds.find(grp->grpId));
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
if( isGxsChannels
&& grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC
&& grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )
@ -241,7 +241,7 @@ bool RsGxsIntegrityCheck::check()
cg.mMeta = meta;
indexedGroups.insert(grp->grpId);
DeepSearch::indexChannelGroup(cg);
DeepChannelsIndex::indexChannelGroup(cg);
}
else
{
@ -256,14 +256,15 @@ bool RsGxsIntegrityCheck::check()
delete rIt;
}
#endif
#endif // def RS_DEEP_CHANNEL_INDEX
}
else
{
grpsToDel.push_back(grp->grpId);
#ifdef RS_DEEP_SEARCH
if(isGxsChannels) DeepSearch::removeChannelFromIndex(grp->grpId);
#endif
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepChannelsIndex::removeChannelFromIndex(grp->grpId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
if( !(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) &&
@ -320,10 +321,10 @@ bool RsGxsIntegrityCheck::check()
if (nxsMsgIt == nxsMsgV.end())
{
msgsToDel[grpId].insert(msgId);
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepSearch::removeChannelPostFromIndex(grpId, msgId);
#endif
DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId);
#endif // def RS_DEEP_CHANNEL_INDEX
}
}
}
@ -348,14 +349,15 @@ bool RsGxsIntegrityCheck::check()
<< " with wrong hash or null meta data. meta="
<< (void*)msg->metaData << std::endl;
msgsToDel[msg->grpId].insert(msg->msgId);
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
if(isGxsChannels)
DeepSearch::removeChannelPostFromIndex(msg->grpId, msg->msgId);
#endif
DeepChannelsIndex::removeChannelPostFromIndex(
msg->grpId, msg->msgId );
#endif // def RS_DEEP_CHANNEL_INDEX
}
else if (subscribed_groups.count(msg->metaData->mGroupId))
{
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
if( isGxsChannels
&& indexedGroups.count(msg->metaData->mGroupId) )
{
@ -373,7 +375,7 @@ bool RsGxsIntegrityCheck::check()
cgIt->toChannelPost(cg, false);
cg.mMeta = meta;
DeepSearch::indexChannelPost(cg);
DeepChannelsIndex::indexChannelPost(cg);
}
else if(dynamic_cast<RsGxsCommentItem*>(rIt)) {}
else if(dynamic_cast<RsGxsVoteItem*>(rIt)) {}
@ -391,7 +393,7 @@ bool RsGxsIntegrityCheck::check()
delete rIt;
}
#endif
#endif // def RS_DEEP_CHANNEL_INDEX
if(!msg->metaData->mAuthorId.isNull())
{

2
libretroshare/src/gxs/rsgxsutil.h
View File

@ -213,7 +213,7 @@ private:
RsGeneralDataService* const mDs;
RsGenExchange *mGenExchangeClient;
#ifdef RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
RsSerialType& mSerializer;
#endif
bool mDone;

19
libretroshare/src/gxstunnel/p3gxstunnel.cc
View File

@ -1596,7 +1596,13 @@ bool p3GxsTunnelService::getTunnelInfo(const RsGxsTunnelId& tunnel_id,GxsTunnelI
// Data packets
info.pending_data_packets = 0;
info.pending_data_packets = 0;
RsPeerId p(tunnel_id);
for(auto it(pendingGxsTunnelDataItems.begin());it!=pendingGxsTunnelDataItems.end();++it)
if(it->second.data_item->PeerId() == p)
++info.pending_data_packets ;
info.total_data_packets_sent=0 ;
info.total_data_packets_received=0 ;
@ -1704,7 +1710,16 @@ bool p3GxsTunnelService::getTunnelsInfo(std::vector<RsGxsTunnelService::GxsTunne
ti.tunnel_status = it->second.status ;
ti.total_size_sent = it->second.total_sent ;
ti.total_size_received = it->second.total_received ;
ti.pending_data_packets = 0;
RsPeerId p(it->first);
for(auto it(pendingGxsTunnelDataItems.begin());it!=pendingGxsTunnelDataItems.end();++it)
if(it->second.data_item->PeerId() == p)
++ti.pending_data_packets ;
ti.total_data_packets_sent =0; // not accounted for yet.
ti.total_data_packets_received=0 ; // not accounted for yet.
infos.push_back(ti) ;
}

36
libretroshare/src/libretroshare.pro
View File

@ -404,9 +404,7 @@ HEADERS += pqi/authssl.h \
pqi/pqiqosstreamer.h \
pqi/sslfns.h \
pqi/pqinetstatebox.h \
pqi/p3servicecontrol.h \
# pqi/p3dhtmgr.h \
pqi/p3servicecontrol.h
HEADERS += rsserver/p3face.h \
rsserver/p3history.h \
@ -653,8 +651,8 @@ equals(RS_UPNP_LIB, miniupnpc) {
contains(RS_UPNP_LIB, upnp) {
HEADERS += rs_upnp/upnp18_retrocompat.h
HEADERS += rs_upnp/UPnPBase.h rs_upnp/upnphandler_linux.h
SOURCES += rs_upnp/UPnPBase.cpp rs_upnp/upnphandler_linux.cc
HEADERS += rs_upnp/UPnPBase.h rs_upnp/upnphandler_libupnp.h
SOURCES += rs_upnp/UPnPBase.cpp rs_upnp/upnphandler_libupnp.cc
}
# new gxs cache system
@ -901,8 +899,32 @@ rs_jsonapi {
SOURCES += jsonapi/jsonapi.cpp
}
rs_deep_search {
HEADERS += deep_search/deep_search.h
rs_deep_channels_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/channelsindex.hpp
SOURCES += deep_search/channelsindex.cpp
}
rs_deep_files_index {
HEADERS *= deep_search/commonutils.hpp
SOURCES *= deep_search/commonutils.cpp
HEADERS += deep_search/filesindex.hpp
SOURCES += deep_search/filesindex.cpp
}
rs_deep_files_index_ogg {
HEADERS += deep_search/filesoggindexer.hpp
}
rs_deep_files_index_flac {
HEADERS += deep_search/filesflacindexer.hpp
}
rs_deep_files_index_taglib {
HEADERS += deep_search/filestaglibindexer.hpp
}
rs_broadcast_discovery {

1795
libretroshare/src/pqi/p3dhtmgr.cc
View File

File diff suppressed because it is too large Load Diff

246
libretroshare/src/pqi/p3dhtmgr.h
View File

@ -1,246 +0,0 @@
/*******************************************************************************
* libretroshare/src/pqi: p3dhtmgr.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2004-2008 by Robert Fernie. *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef MRK_P3_DHT_MANAGER_HEADER
#define MRK_P3_DHT_MANAGER_HEADER
/* Interface class for DHT data */
#include <string>
#include <map>
#include "pqi/pqinetwork.h"
#include "util/rsthreads.h"
#include "pqi/pqimonitor.h"
#include "pqi/pqiassist.h"
/* All other #defs are in .cc */
#define DHT_ADDR_INVALID 0xff
#define DHT_ADDR_TCP 0x01
#define DHT_ADDR_UDP 0x02
/* for DHT peer STATE */
#define DHT_PEER_OFF 0
#define DHT_PEER_INIT 1
#define DHT_PEER_SEARCH 2
#define DHT_PEER_FOUND 3
/* for DHT peer STATE (ownEntry) */
#define DHT_PEER_ADDR_KNOWN 4
#define DHT_PEER_PUBLISHED 5
/* Interface with Real DHT Implementation */
#define DHT_MODE_SEARCH 1
#define DHT_MODE_PUBLISH 1
#define DHT_MODE_NOTIFY 2
#define DHT_MODE_BOOTSTRAP 3
/* TIMEOUTS: Reference Values are set here... */
#define DHT_SEARCH_PERIOD 1800 /* PeerKeys: if we haven't found them: 30 min */
#define DHT_CHECK_PERIOD 1800 /* PeerKeys: re-lookup peer: 30 min */
#define DHT_PUBLISH_PERIOD 1800 /* OwnKey: 30 min */
#define DHT_NOTIFY_PERIOD 300 /* 5 min - Notify Check period */
/* TTLs for DHTs posts */
#define DHT_TTL_PUBLISH (DHT_PUBLISH_PERIOD + 120) // for a little overlap.
#define DHT_TTL_NOTIFY (DHT_NOTIFY_PERIOD + 60) // for time to find it...
#define DHT_TTL_BOOTSTRAP (DHT_PUBLISH_PERIOD) // To start with.
class dhtPeerEntry
{
public:
dhtPeerEntry();
RsPeerId id;
uint32_t state;
rstime_t lastTS;
uint32_t notifyPending;
rstime_t notifyTS;
struct sockaddr_in laddr, raddr;
uint32_t type; /* ADDR_TYPE as defined above */
std::string hash1; /* SHA1 Hash of id */
std::string hash2; /* SHA1 Hash of reverse Id */
};
class p3DhtMgr: public pqiNetAssistConnect, public RsThread
{
/*
*/
public:
p3DhtMgr(RsPeerId id, pqiConnectCb *cb);
/********** External DHT Interface ************************
* These Functions are the external interface
* for the DHT, and must be non-blocking and return quickly
*/
/* OVERLOADED From pqiNetAssistConnect. */
virtual void enable(bool on);
virtual void shutdown();
virtual void restart();
virtual bool getEnabled(); /* on */
virtual bool getActive(); /* actually working */
virtual void setBootstrapAllowed(bool on);
virtual bool getBootstrapAllowed();
/* set key data */
virtual bool setExternalInterface(struct sockaddr_in laddr,
struct sockaddr_in raddr, uint32_t type);
/* add / remove peers */
virtual bool findPeer(const RsPeerId& id);
virtual bool dropPeer(const RsPeerId& id);
/* post DHT key saying we should connect (callback when done) */
virtual bool notifyPeer(const RsPeerId& id);
/* extract current peer status */
virtual bool getPeerStatus(const RsPeerId& id,
struct sockaddr_storage &laddr, struct sockaddr_storage &raddr,
uint32_t &type, uint32_t &mode);
/* stun */
virtual bool enableStun(bool on);
virtual bool addStun(std::string id);
//doneStun();
/********** Higher Level DHT Work Functions ************************
* These functions translate from the strings/addresss to
* key/value pairs.
*/
public:
/* results from DHT proper */
virtual bool dhtResultNotify(std::string id);
virtual bool dhtResultSearch(std::string id,
struct sockaddr_in &laddr, struct sockaddr_in &raddr,
uint32_t type, std::string sign);
virtual bool dhtResultBootstrap(std::string idhash);
protected:
/* can block briefly (called only from thread) */
virtual bool dhtPublish(std::string idhash,
struct sockaddr_in &laddr,
struct sockaddr_in &raddr,
uint32_t type, std::string sign);
virtual bool dhtNotify(std::string idhash, std::string ownIdHash,
std::string sign);
virtual bool dhtSearch(std::string idhash, uint32_t mode);
virtual bool dhtBootstrap(std::string idhash, std::string ownIdHash,
std::string sign); /* to publish bootstrap */
/********** Actual DHT Work Functions ************************
* These involve a very simple LOW-LEVEL interface ...
*
* publish
* search
* result
*
*/
public:
/* Feedback callback (handled here) */
virtual bool resultDHT(std::string key, std::string value);
protected:
virtual bool dhtInit();
virtual bool dhtShutdown();
virtual bool dhtActive();
virtual int status(std::ostream &out);
virtual bool publishDHT(std::string key, std::string value, uint32_t ttl);
virtual bool searchDHT(std::string key);
/********** Internal DHT Threading ************************
*
*/
public:
virtual void run();
private:
/* search scheduling */
void checkDHTStatus();
int checkStunState();
int checkStunState_Active(); /* when in active state */
int doStun();
int checkPeerDHTKeys();
int checkOwnDHTKeys();
int checkNotifyDHT();
void clearDhtData();
/* IP Bootstrap */
bool getDhtBootstrapList();
std::string BootstrapId(uint32_t bin);
std::string randomBootstrapId();
/* other feedback through callback */
// use pqiNetAssistConnect.. version pqiConnectCb *connCb;
/* protected by Mutex */
RsMutex dhtMtx;
bool mDhtOn; /* User desired state */
bool mDhtModifications; /* any user requests? */
dhtPeerEntry ownEntry;
rstime_t ownNotifyTS;
std::map<RsPeerId, dhtPeerEntry> peers;
std::list<std::string> stunIds;
bool mStunRequired;
uint32_t mDhtState;
rstime_t mDhtActiveTS;
bool mBootstrapAllowed;
rstime_t mLastBootstrapListTS;
};
#endif // MRK_P3_DHT_MANAGER_HEADER

1
libretroshare/src/pqi/p3linkmgr.cc
View File

@ -28,7 +28,6 @@
#include "rsserver/p3face.h"
#include "pqi/authssl.h"
#include "pqi/p3dhtmgr.h" // Only need it for constants.
#include "tcponudp/tou.h"
#include "util/extaddrfinder.h"
#include "util/dnsresolver.h"

219
libretroshare/src/pqi/p3netmgr.cc
View File

@ -79,16 +79,12 @@ const uint32_t MAX_UPNP_COMPLETE = 600; /* 10 min... seems to take a while */
// #define NETMGR_DEBUG_TICK 1
// #define NETMGR_DEBUG_STATEBOX 1
pqiNetStatus::pqiNetStatus()
:mLocalAddrOk(false), mExtAddrOk(false), mExtAddrStableOk(false),
mUpnpOk(false), mDhtOk(false), mResetReq(false)
pqiNetStatus::pqiNetStatus() :
mExtAddrOk(false), mExtAddrStableOk(false), mUpnpOk(false), mDhtOk(false),
mDhtNetworkSize(0), mDhtRsNetworkSize(0), mResetReq(false)
{
mDhtNetworkSize = 0;
mDhtRsNetworkSize = 0;
sockaddr_storage_clear(mLocalAddr);
sockaddr_storage_clear(mExtAddr);
return;
}
@ -96,7 +92,6 @@ pqiNetStatus::pqiNetStatus()
void pqiNetStatus::print(std::ostream &out)
{
out << "pqiNetStatus: ";
out << "mLocalAddrOk: " << mLocalAddrOk;
out << " mExtAddrOk: " << mExtAddrOk;
out << " mExtAddrStableOk: " << mExtAddrStableOk;
out << std::endl;
@ -108,14 +103,13 @@ void pqiNetStatus::print(std::ostream &out)
out << std::endl;
out << "mLocalAddr: " << sockaddr_storage_tostring(mLocalAddr) << " ";
out << "mExtAddr: " << sockaddr_storage_tostring(mExtAddr) << " ";
out << " NetOk: " << NetOk();
out << std::endl;
}
p3NetMgrIMPL::p3NetMgrIMPL()
:mPeerMgr(NULL), mLinkMgr(NULL), mNetMtx("p3NetMgr"),
mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false)
p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr),
mNetMtx("p3NetMgr"), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false),
mDoNotNetCheckUntilTs(0)
{
{
@ -130,7 +124,6 @@ p3NetMgrIMPL::p3NetMgrIMPL()
mNetFlags = pqiNetStatus();
mOldNetFlags = pqiNetStatus();
mLastSlowTickTime = 0;
mOldNatType = RSNET_NATTYPE_UNKNOWN;
mOldNatHole = RSNET_NATHOLE_UNKNOWN;
sockaddr_storage_clear(mLocalAddr);
@ -415,47 +408,37 @@ void p3NetMgrIMPL::netStartup()
void p3NetMgrIMPL::tick()
{
rstime_t now = time(NULL);
bool doSlowTick = false;
rstime_t now = time(nullptr);
rstime_t dontCheckNetUntil;
{ RS_STACK_MUTEX(mNetMtx); dontCheckNetUntil = mDoNotNetCheckUntilTs; }
if(now >= dontCheckNetUntil) netStatusTick();
uint32_t netStatus; { RS_STACK_MUTEX(mNetMtx); netStatus = mNetStatus; }
switch (netStatus)
{
RsStackMutex stack(mNetMtx); /************** LOCK MUTEX ***************/
if (now > mLastSlowTickTime)
case RS_NET_LOOPBACK:
if(dontCheckNetUntil <= now)
{
mLastSlowTickTime = now;
doSlowTick = true;
RS_STACK_MUTEX(mNetMtx);
mDoNotNetCheckUntilTs = now + 30;
}
}
if (doSlowTick)
{
slowTick();
}
}
void p3NetMgrIMPL::slowTick()
{
netTick();
netAssistTick();
updateNetStateBox_temporal();
break;
default:
netAssistTick();
updateNetStateBox_temporal();
#ifdef RS_USE_DHT_STUNNER
if (mDhtStunner)
{
mDhtStunner->tick();
}
if (mProxyStunner)
{
mProxyStunner->tick();
}
if (mDhtStunner) mDhtStunner->tick();
if (mProxyStunner) mProxyStunner->tick();
#endif // RS_USE_DHT_STUNNER
break;
}
}
#define STARTUP_DELAY 5
void p3NetMgrIMPL::netTick()
void p3NetMgrIMPL::netStatusTick()
{
#ifdef NETMGR_DEBUG_TICK
@ -986,10 +969,9 @@ bool p3NetMgrIMPL::checkNetAddress()
if (mNetMode & RS_NET_MODE_TRY_LOOPBACK)
{
#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET)
std::cerr << "p3NetMgrIMPL::checkNetAddress() LOOPBACK ... forcing to 127.0.0.1";
std::cerr << std::endl;
#endif
RsInfo() << __PRETTY_FUNCTION__ <<" network mode set to LOOPBACK,"
<< " forcing address to 127.0.0.1" << std::endl;
sockaddr_storage_ipv4_aton(prefAddr, "127.0.0.1");
validAddr = true;
}
@ -1034,137 +1016,77 @@ bool p3NetMgrIMPL::checkNetAddress()
break;
}
}
/* If no satisfactory local address has been found yet relax and
* accept also loopback addresses */
if(!validAddr) for (auto it = addrs.begin(); it!=addrs.end(); ++it)
{
sockaddr_storage& addr(*it);
if( sockaddr_storage_isValidNet(addr) &&
sockaddr_storage_ipv6_to_ipv4(addr) )
{
prefAddr = addr;
validAddr = true;
break;
}
}
}
}
if (!validAddr)
{
Dbg3() << __PRETTY_FUNCTION__ << " no valid local network address found"
<< std::endl;
RsErr() << __PRETTY_FUNCTION__ << " no valid local network address "
<<" found. Report to developers." << std::endl;
print_stacktrace();
return false;
}
/* check addresses */
{
RS_STACK_MUTEX(mNetMtx);
{ RS_STACK_MUTEX(mNetMtx);
sockaddr_storage_copy(mLocalAddr, oldAddr);
addrChanged = !sockaddr_storage_sameip(prefAddr, mLocalAddr);
#ifdef NETMGR_DEBUG_TICK
std::cerr << "p3NetMgrIMPL::checkNetAddress()";
std::cerr << std::endl;
std::cerr << "Current Local: " << sockaddr_storage_tostring(mLocalAddr);
std::cerr << std::endl;
std::cerr << "Current Preferred: " << sockaddr_storage_iptostring(prefAddr);
std::cerr << std::endl;
#endif
#ifdef NETMGR_DEBUG_RESET
if (addrChanged)
{
std::cerr << "p3NetMgrIMPL::checkNetAddress() Address Changed!";
std::cerr << std::endl;
std::cerr << "Current Local: " << sockaddr_storage_tostring(mLocalAddr);
std::cerr << std::endl;
std::cerr << "Current Preferred: " << sockaddr_storage_iptostring(prefAddr);
std::cerr << std::endl;
}
#endif
// update address.
sockaddr_storage_copyip(mLocalAddr, prefAddr);
sockaddr_storage_copy(mLocalAddr, mNetFlags.mLocalAddr);
if(sockaddr_storage_isLoopbackNet(mLocalAddr))
{
#ifdef NETMGR_DEBUG
std::cerr << "p3NetMgrIMPL::checkNetAddress() laddr: Loopback" << std::endl;
#endif
mNetFlags.mLocalAddrOk = false;
mNetStatus = RS_NET_LOOPBACK;
}
else if (!sockaddr_storage_isValidNet(mLocalAddr))
{
#ifdef NETMGR_DEBUG
std::cerr << "p3NetMgrIMPL::checkNetAddress() laddr: invalid" << std::endl;
#endif
mNetFlags.mLocalAddrOk = false;
}
else
{
#ifdef NETMGR_DEBUG_TICK
std::cerr << "p3NetMgrIMPL::checkNetAddress() laddr okay" << std::endl;
#endif
mNetFlags.mLocalAddrOk = true;
}
int port = sockaddr_storage_port(mLocalAddr);
if ((port < PQI_MIN_PORT) || (port > PQI_MAX_PORT))
// Check if local port is valid, reset it if not
if (!sockaddr_storage_port(mLocalAddr))
{
#ifdef NETMGR_DEBUG
std::cerr << "p3NetMgrIMPL::checkNetAddress() Correcting Port to DEFAULT" << std::endl;
#endif
uint16_t new_port = htons(PQI_MIN_PORT_RNG + (RSRandom::random_u32() % (PQI_MAX_PORT - PQI_MIN_PORT_RNG)));
sockaddr_storage_setport(mLocalAddr, new_port);
/* Using same port as external may make some NAT happier */
uint16_t port = sockaddr_storage_port(mExtAddr);
/* Avoid to automatically set a local port to a reserved one < 1024
* that needs special permissions or root access.
* This do not impede the user to set a reserved port manually,
* which make sense in some cases. */
while (port < 1025)
port = static_cast<uint16_t>(RsRandom::random_u32());
sockaddr_storage_setport(mLocalAddr, htons(port));
addrChanged = true;
RsWarn() << __PRETTY_FUNCTION__ << " local port was 0, corrected "
<<"to: " << port << std::endl;
}
#if DEAD_CODE
/* Enabling this piece of code breaks setup where an additional BOFH
* overlooked port like 80 or 443 is manually forwarded to RetroShare to
* avoid restrictive firewals.
* In the case of a real mismatch, it is not really problematic, as our
* peers would get and then attempt to connect also to the right port.
*/
/* if localaddr == serveraddr, then ensure that the ports
* are the same (modify server)... this mismatch can
* occur when the local port is changed....
*/
if ( sockaddr_storage_sameip(mLocalAddr, mExtAddr)
&& sockaddr_storage_port(mLocalAddr) != sockaddr_storage_port(mExtAddr) )
{
#ifdef NETMGR_DEBUG_RESET
std::cerr << __PRETTY_FUNCTION__ << " local and external ports are"
<< " not the same. Setting external port to "
<< sockaddr_storage_port(mLocalAddr) << std::endl;
#endif
sockaddr_storage_setport(mExtAddr, sockaddr_storage_port(mLocalAddr));
addrChanged = true;
}
#endif // DEAD_CODE
#ifdef NETMGR_DEBUG_TICK
std::cerr << __PRETTY_FUNCTION__ << " Final Local Address: "
<< sockaddr_storage_tostring(mLocalAddr) << std::endl;
#endif
}
} // RS_STACK_MUTEX(mNetMtx);
if (addrChanged)
{
#ifdef NETMGR_DEBUG_RESET
std::cerr << "p3NetMgrIMPL::checkNetAddress() local address changed, resetting network." << std::endl;
std::cerr << std::endl;
#endif
RsInfo() << __PRETTY_FUNCTION__ << " local address changed, resetting"
<<" network." << std::endl;
if (mPeerMgr)
{
mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
}
std::cerr << __PRETTY_FUNCTION__
<< " local address changed, resetting network" << std::endl;
if (mPeerMgr) mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr);
netReset();
}
return 1;
return true;
}
@ -2051,3 +1973,6 @@ void p3NetMgrIMPL::updateNetStateBox_reset()
}
p3NetMgr::~p3NetMgr() = default;
pqiNetAssist::~pqiNetAssist() = default;
pqiNetAssistPeerShare::~pqiNetAssistPeerShare() = default;
pqiNetAssistConnect::~pqiNetAssistConnect() = default;

19
libretroshare/src/pqi/p3netmgr.h
View File

@ -44,13 +44,10 @@ class DNSResolver ;
class pqiNetStatus
struct pqiNetStatus
{
public:
pqiNetStatus();
bool mLocalAddrOk; // Local address is not loopback.
bool mExtAddrOk; // have external address.
bool mExtAddrStableOk; // stable external address.
bool mUpnpOk; // upnp is ok.
@ -65,11 +62,6 @@ class pqiNetStatus
bool mResetReq; // Not Used yet!.
void print(std::ostream &out);
bool NetOk() // minimum to believe network is okay.`
{
return (mLocalAddrOk && mExtAddrOk);
}
};
class p3PeerMgr;
@ -211,11 +203,6 @@ void addNetListener(pqiNetListener *listener);
// SHOULD MAKE THIS PROTECTED.
bool checkNetAddress(); /* check our address is sensible */
protected:
void slowTick();
protected:
/****************** Internal Interface *******************/
bool enableNetAssistFirewall(bool on);
@ -248,7 +235,7 @@ bool netAssistAttach(bool on);
void netReset();
void statusTick();
void netTick();
void netStatusTick();
void netStartup();
/* startup the bits */
@ -335,7 +322,7 @@ void netStatusReset_locked();
// Improved NetStatusBox, which uses the Stunners!
pqiNetStateBox mNetStateBox;
rstime_t mLastSlowTickTime;
rstime_t mDoNotNetCheckUntilTs;
uint32_t mOldNatType;
uint32_t mOldNatHole;

15
libretroshare/src/pqi/p3peermgr.cc
View File

@ -3,8 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright (C) 2007-2011 Robert Fernie *
* Copyright (C) 2015-2018 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2007-2011 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2015-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -34,11 +34,6 @@
#include "pqi/p3historymgr.h"
#include "pqi/pqinetwork.h" // for getLocalAddresses
//#include "pqi/p3dhtmgr.h" // Only need it for constants.
//#include "tcponudp/tou.h"
//#include "util/extaddrfinder.h"
//#include "util/dnsresolver.h"
#include "util/rsprint.h"
#include "util/rsstring.h"
#include "util/rsdebug.h"
@ -1175,8 +1170,8 @@ bool p3PeerMgrIMPL::addSslOnlyFriend(
* previously known IP addresses */
if(!dt.isHiddenNode)
{
for(const std::string& locator : dt.ipAddressList)
addPeerLocator(sslId, locator);
for(const std::string& ipStr : dt.ipAddressList)
addPeerLocator(sslId, RsUrl(ipStr));
if(dt.extPort && !dt.extAddr.empty())
{
@ -3159,4 +3154,4 @@ bool p3PeerMgrIMPL::removeUnusedLocations()
return true;
}
p3PeerMgr::~p3PeerMgr() = default;

11
libretroshare/src/pqi/p3peermgr.h
View File

@ -3,7 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2007-2011 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2007-2011 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2015-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -112,8 +113,6 @@ class peerState
class RsNodeGroupItem;
struct RsGroupInfo;
std::string textPeerState(peerState &state);
class p3LinkMgr;
class p3NetMgr;
@ -123,10 +122,6 @@ class p3NetMgrIMPL;
class p3PeerMgr
{
public:
p3PeerMgr() {}
virtual ~p3PeerMgr() {}
virtual bool addFriend( const RsPeerId &ssl_id, const RsPgpId &gpg_id,
uint32_t netMode = RS_NET_MODE_UDP,
uint16_t vsDisc = RS_VS_DISC_FULL,
@ -242,7 +237,7 @@ virtual bool locked_computeCurrentBestOwnExtAddressCandidate(sockaddr_storage
/*************************************************************************************************/
/*************************************************************************************************/
virtual ~p3PeerMgr();
};

6
libretroshare/src/pqi/pqi_base.h
View File

@ -37,12 +37,6 @@ struct RSTrafficClue;
#include "serialiser/rsserial.h"
#include "retroshare/rstypes.h"
#define PQI_MIN_PORT 10 // TO ALLOW USERS TO HAVE PORT 80! - was 1024
#define PQI_MIN_PORT_RNG 1024
#define PQI_MAX_PORT 65535
#define PQI_DEFAULT_PORT 7812
int getPQIsearchId();
int fixme(char *str, int n);

77
libretroshare/src/pqi/pqiassist.h
View File

@ -19,8 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef MRK_PQI_ASSIST_H
#define MRK_PQI_ASSIST_H
#pragma once
#include <string>
#include <map>
@ -28,25 +27,15 @@
#include "pqi/pqinetwork.h"
#include "pqi/pqimonitor.h"
/**********
* This header file provides two interfaces for assisting
* the connections to friends.
*
* pqiNetAssistFirewall - which will provides interfaces
* to functionality like upnp and apple's equivalent.
*
* pqiNetAssistConnect - which will provides interfaces
* to other networks (DHT) etc that can provide information.
* These classes would be expected to use the pqiMonitor
* callback system to notify the connectionMgr.
*
***/
/**
* @file
* This header file provides interfaces for assisting the connections to
* friends.
*/
class pqiNetAssist
{
public:
virtual ~pqiNetAssist() { return; }
public:
/* External Interface */
virtual void enable(bool on) = 0;
@ -58,28 +47,24 @@ virtual bool getActive() = 0;
virtual int tick() { return 0; } /* for internal accounting */
virtual ~pqiNetAssist();
};
#define PFP_TYPE_UDP 0x0001
#define PFP_TYPE_TCP 0x0002
class PortForwardParams
struct PortForwardParams
{
public:
uint32_t fwdId;
uint32_t status;
uint32_t typeFlags;
struct sockaddr_storage intAddr;
struct sockaddr_storage extaddr;
sockaddr_storage intAddr;
sockaddr_storage extaddr;
};
/**
* Provides interfaces to functionality like upnp and apple's equivalent.
*/
class pqiNetAssistFirewall: public pqiNetAssist
{
public:
virtual ~pqiNetAssistFirewall() { return; }
public:
/* the address that the listening port is on */
virtual void setInternalPort(unsigned short iport_in) = 0;
virtual void setExternalPort(unsigned short eport_in) = 0;
@ -102,11 +87,13 @@ virtual bool statusPortForward(const uint32_t fwdId, PortForwardParams &param
class pqiNetAssistPeerShare
{
public:
/* share Addresses for various reasons (bad peers, etc) */
virtual void updatePeer(const RsPeerId& id, const struct sockaddr_storage &addr, int type, int reason, int age) = 0;
public:
/** share Addresses for various reasons (bad peers, etc) */
virtual void updatePeer(
const RsPeerId& id, const struct sockaddr_storage &addr,
int type, int reason, int age ) = 0;
virtual ~pqiNetAssistPeerShare();
};
@ -141,20 +128,22 @@ virtual int tick() = 0; /* for internal accounting */
#define NETASSIST_KNOWN_PEER_TYPE_MASK 0xff00
/**
* Provides interfaces to other networks like DHT that can provide information.
* These classes would be expected to use the pqiMonitor callback system to
* notify the connectionMgr.
*/
class pqiNetAssistConnect: public pqiNetAssist
{
/*
*/
public:
pqiNetAssistConnect(const RsPeerId& id, pqiConnectCb *cb)
:mPeerId(id), mConnCb(cb) { return; }
public:
pqiNetAssistConnect(const RsPeerId& id, pqiConnectCb *cb) :
mPeerId(id), mConnCb(cb) {}
/********** External DHT Interface ************************
* These Functions are the external interface
* for the DHT, and must be non-blocking and return quickly
*/
/* add / remove peers */
virtual bool findPeer(const RsPeerId& id) = 0;
virtual bool dropPeer(const RsPeerId& id) = 0;
@ -175,11 +164,9 @@ virtual bool setAttachMode(bool on) = 0; // FIXUP.
/***** Stats for Network / DHT *****/
virtual bool getNetworkStats(uint32_t &netsize, uint32_t &localnetsize) = 0; // DEPRECIATE.
protected:
~pqiNetAssistConnect() override;
protected:
RsPeerId mPeerId;
pqiConnectCb *mConnCb;
};
#endif /* MRK_PQI_ASSIST_H */

4
libretroshare/src/pqi/pqimonitor.h
View File

@ -63,10 +63,6 @@ const uint32_t RS_STUN_FRIEND = 0x0020;
const uint32_t RS_STUN_FRIEND_OF_FRIEND = 0x0040;
// for the old p3dhtmgr - amazed it still compiles ;)
#define RS_CONNECT_PASSIVE 1
#define RS_CONNECT_ACTIVE 2
#define RS_CB_DHT 0x0001 /* from dht */
#define RS_CB_DISC 0x0002 /* from peers */
#define RS_CB_PERSON 0x0003 /* from connection */

3
libretroshare/src/pqi/pqissl.h
View File

@ -75,9 +75,6 @@ const int WINDOWS_TCP_BUFFER_SIZE = 512 * 1024; // 512 KB
*
*/
class pqissl;
class cert;
class pqissllistener;
class p3LinkMgr;
struct RsPeerCryptoParams;

3
libretroshare/src/pqi/pqisslproxy.h
View File

@ -34,9 +34,6 @@
* It uses the Domain Name interface of SOCKS5, as opposed to an IP address.
*/
class pqisslproxy;
class cert;
/* This provides a NetBinInterface, which is
* primarily inherited from pqissl.
* fns declared here are different -> all others are identical.

40
libretroshare/src/retroshare/rsbroadcastdiscovery.h
View File

@ -107,5 +107,45 @@ public:
*/
virtual std::vector<RsBroadcastDiscoveryResult> getDiscoveredPeers() = 0;
/**
* @brief Check if multicast listening is enabled
* @jsonapi{development}
* On some platforms such as Android multicast listening, which is needed
* for broadcast discovery, is not enabled by default at WiFi driver level
* @see enableMulticastListening, so this method check if it is enabled.
* On platforms that are not expected to have such a limitation this method
* always return true.
* @return true if enabled, false otherwise.
*/
virtual bool isMulticastListeningEnabled() = 0;
/**
* @brief On platforms that need it enable low level multicast listening
* @jsonapi{development}
* On Android and potencially other mobile platforms, WiFi drivers are
* configured by default to discard any packet that is not directed to the
* unicast mac address of the interface, this way they could save some
* battery but breaks anything that is not unicast, such as broadcast
* discovery. On such platforms this method enable low level multicast
* listening so we can receive advertisement from same broadcast domain
* nodes.
* On platforms without such limitation does nothing and always return
* false.
* It is exposed as a public API so the UI can decide the right equilibrium
* between discoverability and battery saving.
* @return true if multicast listening has been enabled due to this call,
* false otherwise.
*/
virtual bool enableMulticastListening() = 0;
/**
* @brief Disable multicast listening
* @jsonapi{development}
* The opposite of @see enableMulticastListening.
* @return true if multicast listening has been disabled due to this call,
* false otherwise.
*/
virtual bool disableMulticastListening() = 0;
virtual ~RsBroadcastDiscovery();
};

7
libretroshare/src/retroshare/rsconfig.h
View File

@ -206,11 +206,11 @@ struct RSTrafficClue : RsSerializable
struct RsConfigNetStatus : RsSerializable
{
RsConfigNetStatus()
RsConfigNetStatus() : netLocalOk(true)
{
localPort = extPort = 0 ;
firewalled = forwardPort = false ;
DHTActive = uPnPActive = netLocalOk = netUpnpOk = netDhtOk = netStunOk = netExtAddressOk = false ;
DHTActive = uPnPActive = netUpnpOk = netDhtOk = netStunOk = netExtAddressOk = false ;
uPnPState = 0 ;
//DHTPeers = 0 ;
netDhtNetSize = netDhtRsNetSize = 0;
@ -235,7 +235,8 @@ struct RsConfigNetStatus : RsSerializable
int uPnPState;
/* Flags for Network Status */
bool netLocalOk; /* That we've talked to someone! */
RS_DEPRECATED
bool netLocalOk; /// As of today it's meaningless
bool netUpnpOk; /* upnp is enabled and active */
bool netDhtOk; /* response from dht */
bool netStunOk; /* recvd stun / udp packets */

3
libretroshare/src/retroshare/rsevents.h
View File

@ -66,6 +66,9 @@ enum class RsEventType : uint32_t
/// Emitted when a peer state changes, @see RsPeers
PEER_STATE_CHANGED = 6,
/// @see RsMailStatusEvent
MAIL_STATUS_CHANGE = 7,
MAX /// Used to detect invalid event type passed
};

73
libretroshare/src/retroshare/rsfiles.h
View File

@ -202,6 +202,52 @@ struct BannedFileEntry : RsSerializable
}
};
struct DeepFilesSearchResult;
struct TurtleFileInfoV2 : RsSerializable
{
TurtleFileInfoV2() : fSize(0), fWeight(0) {}
TurtleFileInfoV2(const TurtleFileInfo& oldInfo) :
fSize(oldInfo.size), fHash(oldInfo.hash), fName(oldInfo.name),
fWeight(0) {}
#ifdef RS_DEEP_FILES_INDEX
TurtleFileInfoV2(const DeepFilesSearchResult& dRes);
#endif // def RS_DEEP_FILES_INDEX
uint64_t fSize; /// File size
RsFileHash fHash; /// File hash
std::string fName; /// File name
/** @brief Xapian weight of the file which matched the search criteria
* This field is optional (its value is 0 when not specified).
* Given that Xapian weight for the same file is usually different on
* different nodes, it should not be used as an absolute refence, but just
* as an hint of how much the given file match the search criteria.
*/
float fWeight;
/** @brief Xapian snippet of the file which matched the search criteria
* This field is optional (its value is an empty string when not specified).
*/
std::string fSnippet;
/// @see RsSerializable::serial_process
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(fSize);
RS_SERIAL_PROCESS(fHash);
RS_SERIAL_PROCESS(fName);
RS_SERIAL_PROCESS(fWeight);
RS_SERIAL_PROCESS(fSnippet);
}
~TurtleFileInfoV2() override;
};
class RsFiles
{
public:
@ -209,7 +255,7 @@ public:
virtual ~RsFiles() {}
/**
* @brief Provides file data for the gui, media streaming or rpc clients.
* @brief Provides file data for the GUI, media streaming or API clients.
* It may return unverified chunks. This allows streaming without having to
* wait for hashes or completion of the file.
* This function returns an unspecified amount of bytes. Either as much data
@ -217,8 +263,8 @@ public:
* To get more data, call this function repeatedly with different offsets.
*
* @jsonapi{development,manualwrapper}
* note the missing @ the wrapper for this is written manually not
* autogenerated @see JsonApiServer.
* note the wrapper for this is written manually not autogenerated
* @see JsonApiServer.
*
* @param[in] hash hash of the file. The file has to be available on this node
* or it has to be in downloading state.
@ -356,7 +402,9 @@ public:
/**
* @brief Request remote files search
* @jsonapi{development}
* @param[in] matchString string to look for in the search
* @param[in] matchString string to look for in the search. If files deep
* indexing is enabled at compile time support advanced features described
* at https://xapian.org/docs/queryparser.html
* @param multiCallback function that will be called each time a search
* result is received
* @param[in] maxWait maximum wait time in seconds for search results
@ -364,7 +412,7 @@ public:
*/
virtual bool turtleSearchRequest(
const std::string& matchString,
const std::function<void (const std::list<TurtleFileInfo>& results)>& multiCallback,
const std::function<void (const std::vector<TurtleFileInfoV2>& results)>& multiCallback,
rstime_t maxWait = 300 ) = 0;
virtual TurtleRequestId turtleSearch(const std::string& string_to_match) = 0;
@ -627,8 +675,19 @@ public:
*/
virtual bool removeSharedDirectory(std::string dir) = 0;
virtual bool getIgnoreLists(std::list<std::string>& ignored_prefixes, std::list<std::string>& ignored_suffixes,uint32_t& flags) =0;
virtual void setIgnoreLists(const std::list<std::string>& ignored_prefixes, const std::list<std::string>& ignored_suffixes,uint32_t flags) =0;
/**
* @brief Get list of ignored file name prefixes and suffixes
* @param[out] ignoredPrefixes storage for ingored prefixes
* @param[out] ignoredSuffixes storage for ingored suffixes
* @param flags RS_FILE_SHARE_FLAGS_IGNORE_*
* @return false if something failed, true otherwhise
*/
virtual bool getIgnoreLists(
std::list<std::string>& ignoredPrefixes,
std::list<std::string>& ignoredSuffixes,
uint32_t& flags ) = 0;
virtual void setIgnoreLists(const std::list<std::string>& ignored_prefixes, const std::list<std::string>& ignored_suffixes,uint32_t flags) =0;
virtual void setWatchPeriod(int minutes) =0;
virtual void setWatchEnabled(bool b) =0;

18
libretroshare/src/retroshare/rsgxschannels.h
View File

@ -47,7 +47,7 @@ extern RsGxsChannels* rsGxsChannels;
struct RsGxsChannelGroup : RsSerializable
{
RsGxsChannelGroup() : mAutoDownload(false) {}
RsGxsChannelGroup() : mAutoDownload(false) {}
RsGroupMetaData mMeta;
std::string mDescription;
@ -56,14 +56,17 @@ struct RsGxsChannelGroup : RsSerializable
bool mAutoDownload;
/// @see RsSerializable
virtual void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
virtual void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mMeta);
RS_SERIAL_PROCESS(mDescription);
RS_SERIAL_PROCESS(mImage);
RS_SERIAL_PROCESS(mAutoDownload);
}
~RsGxsChannelGroup() override;
};
struct RsGxsChannelPost : RsSerializable
@ -82,8 +85,9 @@ struct RsGxsChannelPost : RsSerializable
RsGxsImage mThumbnail;
/// @see RsSerializable
virtual void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
virtual void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override
{
RS_SERIAL_PROCESS(mMeta);
RS_SERIAL_PROCESS(mOlderVersions);
@ -94,6 +98,8 @@ struct RsGxsChannelPost : RsSerializable
RS_SERIAL_PROCESS(mSize);
RS_SERIAL_PROCESS(mThumbnail);
}
~RsGxsChannelPost() override;
};
@ -544,5 +550,5 @@ public:
virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0;
//////////////////////////////////////////////////////////////////////////////
virtual ~RsGxsChannels();
~RsGxsChannels() override;
};

7
libretroshare/src/retroshare/rsgxstrans.h
View File

@ -23,9 +23,12 @@
#include "retroshare/rstokenservice.h"
#include "retroshare/rsgxsifacehelper.h"
#include "retroshare/rsgxscommon.h"
#include "rsitems/rsserviceids.h"
/// Subservices identifiers (like port for TCP)
enum class GxsTransSubServices : uint16_t
/** Subservices identifiers (like port for TCP)
* @deprecated pay special attention fixing this as it may break
* retro-compatibility */
enum class RS_DEPRECATED_FOR(RsServiceType) GxsTransSubServices : uint16_t
{
UNKNOWN = 0x00,
TEST_SERVICE = 0x01,

13
libretroshare/src/retroshare/rsidentity.h
View File

@ -338,7 +338,7 @@ struct RsIdentityDetails : RsSerializable
RS_SERIAL_PROCESS(mNickname);
RS_SERIAL_PROCESS(mFlags);
RS_SERIAL_PROCESS(mPgpId);
//RS_SERIAL_PROCESS(mReputation);
RS_SERIAL_PROCESS(mReputation);
RS_SERIAL_PROCESS(mAvatar);
RS_SERIAL_PROCESS(mPublishTS);
RS_SERIAL_PROCESS(mLastUsageTS);
@ -352,7 +352,6 @@ struct RsIdentityDetails : RsSerializable
struct RsIdentity : RsGxsIfaceHelper
{
explicit RsIdentity(RsGxsIface& gxs) : RsGxsIfaceHelper(gxs) {}
virtual ~RsIdentity() {}
/**
* @brief Create a new identity
@ -419,6 +418,14 @@ struct RsIdentity : RsGxsIfaceHelper
*/
virtual bool getOwnPseudonimousIds(std::vector<RsGxsId>& ids) = 0;
/**
* @brief Check if an id is known
* @jsonapi{development}
* @param[in] id Id to check
* @return true if the id is known, false otherwise
*/
virtual bool isKnownId(const RsGxsId& id) = 0;
/**
* @brief Check if an id is own
* @jsonapi{development}
@ -568,4 +575,6 @@ struct RsIdentity : RsGxsIfaceHelper
RS_DEPRECATED_FOR("getIdentitiesSummaries getIdentitiesInfo")
virtual bool getGroupData( const uint32_t& token,
std::vector<RsGxsIdGroup>& groups) = 0;
virtual ~RsIdentity();
};

184
libretroshare/src/retroshare/rsmsgs.h
View File

@ -3,7 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2007-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2007-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +20,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RS_MSG_GUI_INTERFACE_H
#define RS_MSG_GUI_INTERFACE_H
#pragma once
#include <list>
#include <iostream>
@ -28,8 +28,11 @@
#include <set>
#include <assert.h>
#include "rstypes.h"
#include "rsgxsifacetypes.h"
#include "retroshare/rstypes.h"
#include "retroshare/rsgxsifacetypes.h"
#include "retroshare/rsevents.h"
#include "util/rsdeprecate.h"
#include "util/rsmemory.h"
/********************** For Messages and Channels *****************/
@ -93,9 +96,31 @@ const ChatLobbyFlags RS_CHAT_LOBBY_FLAGS_PGP_SIGNED ( 0x00000010 ) ; // requi
typedef uint64_t ChatLobbyId ;
typedef uint64_t ChatLobbyMsgId ;
typedef std::string ChatLobbyNickName ;
typedef std::string RsMailMessageId; // should be uint32_t !!
typedef uint64_t MessageId ;
typedef std::string RsMailMessageId; // TODO: rebase on t_RsGenericIdType
/**
* Used to return a tracker id so the API user can keep track of sent mail
* status, it contains mail id, and recipient id
*/
struct RsMailIdRecipientIdPair : RsSerializable
{
RsMailIdRecipientIdPair(RsMailMessageId mailId, RsGxsId recipientId):
mMailId(mailId), mRecipientId(recipientId) {}
RsMailMessageId mMailId;
RsGxsId mRecipientId;
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext &ctx ) override;
bool operator<(const RsMailIdRecipientIdPair& other) const;
bool operator==(const RsMailIdRecipientIdPair& other) const;
RsMailIdRecipientIdPair() = default;
~RsMailIdRecipientIdPair() override = default;
};
namespace Rs
{
@ -137,38 +162,9 @@ class MsgAddress
std::string _addr_string ;
};
class MessageInfo_v2
{
public:
//MessageInfo_v2() {}
unsigned int msgflags;
//RsMessageId msgId;
MsgAddress from ;
std::list<MsgAddress> rcpt ;
// Headers
//
std::string subject;
std::string msg;
rstime_t time_stamp ;
//std::list<MessageHeader> headers ;
std::string attach_title;
std::string attach_comment;
std::list<FileInfo> files;
int size; /* total of files */
int count; /* file count */
};
struct MessageInfo : RsSerializable
{
MessageInfo(): msgflags(0), size(0), count(0), ts(0) {}
virtual ~MessageInfo() = default;
std::string msgId;
@ -202,7 +198,10 @@ struct MessageInfo : RsSerializable
int ts;
// RsSerializable interface
void serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext &ctx) {
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext &ctx ) override
{
RS_SERIAL_PROCESS(msgId);
RS_SERIAL_PROCESS(rspeerid_srcId);
@ -230,26 +229,30 @@ struct MessageInfo : RsSerializable
RS_SERIAL_PROCESS(ts);
}
~MessageInfo() override;
};
struct MsgInfoSummary : RsSerializable
{
MsgInfoSummary() : msgflags(0), count(0), ts(0) {}
virtual ~MsgInfoSummary() = default;
RsMailMessageId msgId;
RsMailMessageId msgId;
RsPeerId srcId;
uint32_t msgflags;
std::list<uint32_t> msgtags; // that leaves 25 bits for user-defined tags.
std::list<uint32_t> msgtags; /// that leaves 25 bits for user-defined tags.
std::string title;
int count; /* file count */
int count; /** file count */
rstime_t ts;
// RsSerializable interface
void serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext &ctx) {
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext &ctx) override
{
RS_SERIAL_PROCESS(msgId);
RS_SERIAL_PROCESS(srcId);
@ -260,6 +263,8 @@ struct MsgInfoSummary : RsSerializable
RS_SERIAL_PROCESS(count);
RS_SERIAL_PROCESS(ts);
}
~MsgInfoSummary() override;
};
struct MsgTagInfo : RsSerializable
@ -291,6 +296,22 @@ struct MsgTagType : RsSerializable
} //namespace Rs
} //namespace Msgs
struct RsMailStatusEvent : RsEvent
{
RsMailStatusEvent() : RsEvent(RsEventType::MAIL_STATUS_CHANGE) {}
std::set<RsMailMessageId> mChangedMsgIds;
/// @see RsEvent
void serial_process( RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx) override
{
RsEvent::serial_process(j, ctx);
RS_SERIAL_PROCESS(mChangedMsgIds);
}
~RsMailStatusEvent() override;
};
#define RS_CHAT_PUBLIC 0x0001
#define RS_CHAT_PRIVATE 0x0002
@ -322,10 +343,13 @@ struct MsgTagType : RsSerializable
struct DistantChatPeerInfo
{
DistantChatPeerInfo() : status(0),pending_items(0) {}
RsGxsId to_id ;
RsGxsId own_id ;
DistantChatPeerId peer_id ; // this is the tunnel id actually
uint32_t status ; // see the values in rsmsgs.h
uint32_t status ; // see the values in rsmsgs.h
uint32_t pending_items; // items not sent, waiting for a tunnel
};
// Identifier for an chat endpoint like
@ -426,11 +450,10 @@ public:
}
};
class VisibleChatLobbyRecord : RsSerializable
struct VisibleChatLobbyRecord : RsSerializable
{
public:
VisibleChatLobbyRecord(): lobby_id(0), total_number_of_peers(0), last_report_time(0){}
virtual ~VisibleChatLobbyRecord() = default;
VisibleChatLobbyRecord():
lobby_id(0), total_number_of_peers(0), last_report_time(0) {}
ChatLobbyId lobby_id ; // unique id of the lobby
std::string lobby_name ; // name to use for this lobby
@ -441,9 +464,11 @@ public:
rstime_t last_report_time ; // last time the lobby was reported.
ChatLobbyFlags lobby_flags ; // see RS_CHAT_LOBBY_PRIVACY_LEVEL_PUBLIC / RS_CHAT_LOBBY_PRIVACY_LEVEL_PRIVATE
// RsSerializable interface
public:
void serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext &ctx) {
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext &ctx) override
{
RS_SERIAL_PROCESS(lobby_id);
RS_SERIAL_PROCESS(lobby_name);
RS_SERIAL_PROCESS(lobby_topic);
@ -453,6 +478,8 @@ public:
RS_SERIAL_PROCESS(last_report_time);
RS_SERIAL_PROCESS(lobby_flags);
}
~VisibleChatLobbyRecord() override;
};
class ChatLobbyInfo : RsSerializable
@ -499,13 +526,6 @@ class RsMsgs
{
public:
RsMsgs() {}
virtual ~RsMsgs() = default;
/****************************************/
/* Message Items */
/****************************************/
/**
* @brief getMessageSummaries
* @jsonapi{development}
@ -523,6 +543,33 @@ public:
*/
virtual bool getMessage(const std::string &msgId, Rs::Msgs::MessageInfo &msg) = 0;
/**
* @brief sendMail
* @jsonapi{development}
* @param[in] from GXS id of the author
* @param[in] subject Mail subject
* @param[in] mailBody Mail body
* @param[in] to list of To: recipients
* @param[in] cc list of CC: recipients
* @param[in] bcc list of BCC: recipients
* @param[in] attachments list of suggested files
* @param[out] trackingIds storage for tracking ids for each sent mail
* @param[out] errorMsg error message if errors occurred, empty otherwise
* @return number of successfully sent mails
*/
virtual uint32_t sendMail(
const RsGxsId from,
const std::string& subject,
const std::string& mailBody,
const std::set<RsGxsId>& to = std::set<RsGxsId>(),
const std::set<RsGxsId>& cc = std::set<RsGxsId>(),
const std::set<RsGxsId>& bcc = std::set<RsGxsId>(),
const std::vector<FileInfo>& attachments = std::vector<FileInfo>(),
std::set<RsMailIdRecipientIdPair>& trackingIds =
RS_DEFAULT_STORAGE_PARAM(std::set<RsMailIdRecipientIdPair>),
std::string& errorMsg =
RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0;
/**
* @brief getMessageCount
* @jsonapi{development}
@ -535,14 +582,6 @@ public:
*/
virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox) = 0;
/**
* @brief MessageSend
* @jsonapi{development}
* @param[in] info
* @return always true
*/
virtual bool MessageSend(Rs::Msgs::MessageInfo &info) = 0;
/**
* @brief SystemMessage
* @jsonapi{development}
@ -916,7 +955,14 @@ virtual bool initiateDistantChatConnexion(
virtual bool getDistantChatStatus(const DistantChatPeerId& pid,DistantChatPeerInfo& info)=0;
virtual bool closeDistantChatConnexion(const DistantChatPeerId& pid)=0;
/**
* @brief MessageSend
* @jsonapi{development}
* @param[in] info
* @return always true
*/
RS_DEPRECATED_FOR(sendMail)
virtual bool MessageSend(Rs::Msgs::MessageInfo &info) = 0;
virtual ~RsMsgs();
};
#endif

3
libretroshare/src/retroshare/rsreputations.h
View File

@ -72,7 +72,6 @@ struct RsReputationInfo : RsSerializable
mFriendsNegativeVotes(0),
mFriendAverageScore(RS_REPUTATION_THRESHOLD_DEFAULT),
mOverallReputationLevel(RsReputationLevel::NEUTRAL) {}
virtual ~RsReputationInfo() {}
RsOpinion mOwnOpinion;
@ -94,6 +93,8 @@ struct RsReputationInfo : RsSerializable
RS_SERIAL_PROCESS(mFriendAverageScore);
RS_SERIAL_PROCESS(mOverallReputationLevel);
}
virtual ~RsReputationInfo();
};

15
libretroshare/src/retroshare/rsturtle.h
View File

@ -44,12 +44,10 @@ extern RsTurtle* rsTurtle;
typedef uint32_t TurtleRequestId ;
typedef RsPeerId TurtleVirtualPeerId;
/**
* This is the structure used to send back results of the turtle search,
* to other peers, to the notifyBase class, to the search caller or to the GUI.
*/
struct TurtleFileInfo : RsSerializable
{
TurtleFileInfo() : size(0) {}
uint64_t size; /// File size
RsFileHash hash; /// File hash
std::string name; /// File name
@ -65,7 +63,7 @@ struct TurtleFileInfo : RsSerializable
RsTypeSerializer::serial_process(
j, ctx, TLV_TYPE_STR_NAME, name, "name" );
}
};
} RS_DEPRECATED_FOR(TurtleFileInfoV2);
struct TurtleTunnelRequestDisplayInfo
{
@ -120,10 +118,9 @@ public:
virtual void setSessionEnabled(bool) = 0 ;
virtual bool sessionEnabled() const = 0 ;
// Lauches a search request through the pipes, and immediately returns
// the request id, which will be further used by the gui to store results
// as they come back.
//
/** Lauches a search request through the pipes, and immediately returns
* the request id, which will be further used by client services to
* handle results as they come back. */
virtual TurtleRequestId turtleSearch(
unsigned char *search_bin_data, uint32_t search_bin_data_len,
RsTurtleClientService* client_service ) = 0;

13
libretroshare/src/retroshare/rstypes.h
View File

@ -399,11 +399,20 @@ struct FileChunksInfo : RsSerializable
CHUNK_STRATEGY_PROGRESSIVE
};
struct SliceInfo
struct SliceInfo : RsSerializable
{
uint32_t start;
uint32_t size;
RsPeerId peer_id;
/// @see RsSerializable
void serial_process(RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx) override
{
RS_SERIAL_PROCESS(start);
RS_SERIAL_PROCESS(size);
RS_SERIAL_PROCESS(peer_id);
}
};
uint64_t file_size; /// real size of the file
@ -432,7 +441,7 @@ struct FileChunksInfo : RsSerializable
RS_SERIAL_PROCESS(chunks);
RS_SERIAL_PROCESS(compressed_peer_availability_maps);
RS_SERIAL_PROCESS(active_chunks);
//RS_SERIAL_PROCESS(pending_slices);
RS_SERIAL_PROCESS(pending_slices);
}
};

4
libretroshare/src/rs_upnp/upnphandler_linux.cc → libretroshare/src/rs_upnp/upnphandler_libupnp.cc
View File

@ -1,5 +1,5 @@
/*******************************************************************************
* libretroshare/src/upnp: upnphandler_linux.cc *
* libretroshare/src/upnp: upnphandler_libupnp.cc *
* *
* libretroshare: retroshare core library *
* *
@ -28,7 +28,7 @@ extern "C" {
#endif
/* This stuff is actually C */
#include "rs_upnp/upnphandler_linux.h"
#include "rs_upnp/upnphandler_libupnp.h"
#include "util/rsnet.h"

7
libretroshare/src/rs_upnp/upnphandler_linux.h → libretroshare/src/rs_upnp/upnphandler_libupnp.h
View File

@ -1,5 +1,5 @@
/*******************************************************************************
* libretroshare/src/upnp: upnphandler_linux.h *
* libretroshare/src/upnp: upnphandler_libupnp.h *
* *
* libretroshare: retroshare core library *
* *
@ -19,8 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef _RS_UPNP_IFACE_H
#define _RS_UPNP_IFACE_H
#pragma once
#include <string.h>
@ -99,5 +98,3 @@ class upnphandler: public pqiNetAssistFirewall
/* info from upnp */
int CtrlPointCallbackEventHandler(Upnp_EventType ,void* , void*);
#endif /* _RS_UPNP_IFACE_H */

7
libretroshare/src/rs_upnp/upnphandler_miniupnp.h
View File

@ -19,12 +19,9 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
//windows/osx (miniupnpc) implementation
#ifndef _RS_UPNP_IFACE_H
#define _RS_UPNP_IFACE_H
#pragma once
#include <string.h>
#include <string>
#include <map>
@ -127,5 +124,3 @@ bool checkUPnPActive();
std::list<upnpforward> activeForwards;
};
#endif /* _RS_UPNP_IFACE_H */

48
libretroshare/src/rsserver/p3msgs.cc
View File

@ -3,7 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2004-2006 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2004-2006 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -20,6 +21,7 @@
* *
*******************************************************************************/
#include <iostream>
#include <tuple>
#include "util/rsdir.h"
#include "util/rsdebug.h"
@ -38,8 +40,7 @@
using namespace Rs::Msgs;
/* external reference point */
RsMsgs *rsMsgs = NULL;
/*extern*/ RsMsgs* rsMsgs = nullptr;
/****************************************/
/****************************************/
@ -300,6 +301,22 @@ bool p3Msgs::MessageSend(MessageInfo &info)
return mMsgSrv->MessageSend(info);
}
uint32_t p3Msgs::sendMail(
const RsGxsId from,
const std::string& subject,
const std::string& body,
const std::set<RsGxsId>& to,
const std::set<RsGxsId>& cc,
const std::set<RsGxsId>& bcc,
const std::vector<FileInfo>& attachments,
std::set<RsMailIdRecipientIdPair>& trackingIds,
std::string& errorMsg )
{
return mMsgSrv->sendMail(
from, subject, body, to, cc, bcc, attachments,
trackingIds, errorMsg );
}
bool p3Msgs::SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag)
{
return mMsgSrv->SystemMessage(title, message, systemFlag);
@ -541,3 +558,28 @@ uint32_t p3Msgs::getDistantChatPermissionFlags()
return mChatSrv->getDistantChatPermissionFlags() ;
}
RsMsgs::~RsMsgs() = default;
RsMailStatusEvent::~RsMailStatusEvent() = default;
Rs::Msgs::MessageInfo::~MessageInfo() = default;
MsgInfoSummary::~MsgInfoSummary() = default;
VisibleChatLobbyRecord::~VisibleChatLobbyRecord() = default;
void RsMailIdRecipientIdPair::serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx )
{
RS_SERIAL_PROCESS(mMailId);
RS_SERIAL_PROCESS(mRecipientId);
}
bool RsMailIdRecipientIdPair::operator<(const RsMailIdRecipientIdPair& o) const
{
return std::tie( mMailId, mRecipientId) <
std::tie(o.mMailId, o.mRecipientId);
}
bool RsMailIdRecipientIdPair::operator==(const RsMailIdRecipientIdPair& o) const
{
return std::tie( mMailId, mRecipientId) ==
std::tie(o.mMailId, o.mRecipientId);
}

32
libretroshare/src/rsserver/p3msgs.h
View File

@ -3,7 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2007-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2007-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -19,8 +20,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#ifndef RS_P3MSG_INTERFACE_H
#define RS_P3MSG_INTERFACE_H
#pragma once
#include "retroshare/rsmsgs.h"
#include "retroshare/rsgxsifacetypes.h"
@ -38,11 +38,25 @@ class RsChatMsgItem;
*/
class p3Msgs: public RsMsgs
{
public:
public:
p3Msgs(p3MsgService *p3m, p3ChatService *p3c)
:mMsgSrv(p3m), mChatSrv(p3c) { return; }
virtual ~p3Msgs() { return; }
p3Msgs(p3MsgService *p3m, p3ChatService *p3c) :
mMsgSrv(p3m), mChatSrv(p3c) {}
~p3Msgs() override = default;
/// @see RsMsgs
uint32_t sendMail(
const RsGxsId from,
const std::string& subject,
const std::string& body,
const std::set<RsGxsId>& to = std::set<RsGxsId>(),
const std::set<RsGxsId>& cc = std::set<RsGxsId>(),
const std::set<RsGxsId>& bcc = std::set<RsGxsId>(),
const std::vector<FileInfo>& attachments = std::vector<FileInfo>(),
std::set<RsMailIdRecipientIdPair>& trackingIds =
RS_DEFAULT_STORAGE_PARAM(std::set<RsMailIdRecipientIdPair>),
std::string& errorMsg =
RS_DEFAULT_STORAGE_PARAM(std::string) ) override;
/****************************************/
/* Message Items */
@ -54,6 +68,7 @@ class p3Msgs: public RsMsgs
virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg);
virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox);
RS_DEPRECATED_FOR(sendMail)
virtual bool MessageSend(Rs::Msgs::MessageInfo &info);
virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag);
virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId);
@ -172,6 +187,3 @@ class p3Msgs: public RsMsgs
p3MsgService *mMsgSrv;
p3ChatService *mChatSrv;
};
#endif

1
libretroshare/src/rsserver/p3serverconfig.cc
View File

@ -160,7 +160,6 @@ int p3ServerConfig::getConfigNetStatus(RsConfigNetStatus &status)
pqiNetStatus nstatus;
mNetMgr->getNetStatus(nstatus);
status.netLocalOk = nstatus.mLocalAddrOk;
status.netUpnpOk = nstatus.mUpnpOk;
status.netStunOk = false;
status.netExtAddressOk = nstatus.mExtAddrOk;

2
libretroshare/src/rsserver/rsinit.cc
View File

@ -702,7 +702,7 @@ RsGRouter *rsGRouter = NULL ;
#include "util/rsrandom.h"
#ifdef RS_USE_LIBUPNP
# include "rs_upnp/upnphandler_linux.h"
# include "rs_upnp/upnphandler_libupnp.h"
#else // def RS_USE_LIBUPNP
# include "rs_upnp/upnphandler_miniupnp.h"
#endif // def RS_USE_LIBUPNP

11
libretroshare/src/serialiser/rsserializer.cc
View File

@ -27,6 +27,7 @@
#include "serialiser/rsserializer.h"
#include "serialiser/rstypeserializer.h"
#include "util/stacktrace.h"
#include "util/rsdebug.h"
const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_NONE ( 0x0000 );
const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_CONFIG ( 0x0001 );
@ -36,6 +37,16 @@ const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_YIELDING ( 0
RsItem *RsServiceSerializer::deserialise(void *data, uint32_t *size)
{
if(!data || !size || !*size)
{
RsErr() << __PRETTY_FUNCTION__ << " Called with null paramethers data: "
<< data << " size: " << static_cast<void*>(size) << " *size: "
<< (size ? *size : 0) << " this should never happen!"
<< std::endl;
print_stacktrace();
return nullptr;
}
if(mFlags & SERIALIZATION_FLAG_SKIP_HEADER)
{
std::cerr << "(EE) Cannot deserialise item with flags SERIALIZATION_FLAG_SKIP_HEADER. Check your code!" << std::endl;

105
libretroshare/src/services/broadcastdiscoveryservice.cc
View File

@ -25,6 +25,10 @@
#include <vector>
#include <iostream>
#ifdef __ANDROID__
# include <QtAndroid>
#endif // def __ANDROID__
#include "services/broadcastdiscoveryservice.h"
#include "retroshare/rspeers.h"
#include "serialiser/rsserializable.h"
@ -94,6 +98,12 @@ BroadcastDiscoveryService::BroadcastDiscoveryService(
{
if(mRsPeers.isHiddenNode(mRsPeers.getOwnId())) return;
#ifdef __ANDROID__
createMulticastLock();
#endif // def __ANDROID__
enableMulticastListening();
mUdcParameters.set_can_discover(true);
mUdcParameters.set_can_be_discovered(true);
mUdcParameters.set_port(port);
@ -104,7 +114,10 @@ BroadcastDiscoveryService::BroadcastDiscoveryService(
}
BroadcastDiscoveryService::~BroadcastDiscoveryService()
{ mUdcPeer.Stop(true); }
{
mUdcPeer.Stop(true);
disableMulticastListening();
}
std::vector<RsBroadcastDiscoveryResult>
BroadcastDiscoveryService::getDiscoveredPeers()
@ -192,6 +205,7 @@ RsBroadcastDiscoveryResult BroadcastDiscoveryService::createResult(
BroadcastDiscoveryPack::fromSerializedString(uData);
RsBroadcastDiscoveryResult rbdr;
rbdr.mPgpFingerprint = bdp.mPgpFingerprint;
rbdr.mSslId = bdp.mSslId;
rbdr.mProfileName = bdp.mProfileName;
rbdr.mLocator.
@ -202,6 +216,95 @@ RsBroadcastDiscoveryResult BroadcastDiscoveryService::createResult(
return rbdr;
}
bool BroadcastDiscoveryService::isMulticastListeningEnabled()
{
#ifdef __ANDROID__
return assertMulticastLockIsvalid() &&
mWifiMulticastLock.callMethod<jboolean>("isHeld");
#endif // def __ANDROID__
return true;
}
bool BroadcastDiscoveryService::enableMulticastListening()
{
#ifdef __ANDROID__
if(assertMulticastLockIsvalid() && !isMulticastListeningEnabled())
{
mWifiMulticastLock.callMethod<void>("acquire");
return true;
}
#endif // def __ANDROID__
return false;
}
bool BroadcastDiscoveryService::disableMulticastListening()
{
#ifdef __ANDROID__
if(assertMulticastLockIsvalid() && isMulticastListeningEnabled())
{
mWifiMulticastLock.callMethod<void>("release");
return true;
}
#endif // def __ANDROID__
return false;
}
#ifdef __ANDROID__
bool BroadcastDiscoveryService::createMulticastLock()
{
Dbg2() << __PRETTY_FUNCTION__ << std::endl;
constexpr auto fname = __PRETTY_FUNCTION__;
const auto failure = [&](const std::string& err)
{
RsErr() << fname << " " << err << std::endl;
return false;
};
if(mWifiMulticastLock.isValid())
return failure("mWifiMulticastLock is already initialized");
QAndroidJniObject context = QtAndroid::androidContext();
if(!context.isValid())
return failure("Cannot retrieve Android context");
QAndroidJniObject WIFI_SERVICE = QAndroidJniObject::getStaticObjectField(
"android.content.Context", "WIFI_SERVICE", "Ljava/lang/String;");
if(!WIFI_SERVICE.isValid())
return failure("Cannot retrieve Context.WIFI_SERVICE value");
QAndroidJniObject wifiManager = context.callObjectMethod(
"getSystemService", "(Ljava/lang/String;)Ljava/lang/Object;",
WIFI_SERVICE.object<jstring>() );
if(!wifiManager.isValid())
return failure("Cannot retrieve Android Wifi Manager");
mWifiMulticastLock = wifiManager.callObjectMethod(
"createMulticastLock",
"(Ljava/lang/String;)Landroid/net/wifi/WifiManager$MulticastLock;",
QAndroidJniObject::fromString(fname).object<jstring>() );
if(!mWifiMulticastLock.isValid())
return failure("Cannot create WifiManager.MulticastLock");
return true;
}
bool BroadcastDiscoveryService::assertMulticastLockIsvalid()
{
if(!mWifiMulticastLock.isValid())
{
RsErr() << __PRETTY_FUNCTION__ << " mWifiMulticastLock is invalid!"
<< std::endl;
print_stacktrace();
return false;
}
return true;
}
#endif // def __ANDROID__
RsBroadcastDiscovery::~RsBroadcastDiscovery() = default;
RsBroadcastDiscoveryResult::~RsBroadcastDiscoveryResult() = default;
RsBroadcastDiscoveryPeerFoundEvent::~RsBroadcastDiscoveryPeerFoundEvent() = default;

35
libretroshare/src/services/broadcastdiscoveryservice.h
View File

@ -27,8 +27,13 @@
#include <udp_discovery_peer.hpp>
#ifdef __ANDROID__
# include <QtAndroidExtras/QAndroidJniObject>
#endif // def __ANDROID__
#include "retroshare/rsbroadcastdiscovery.h"
#include "util/rsthreads.h"
#include "util/rsdebug.h"
namespace UDC = udpdiscovery;
class RsPeers;
@ -37,13 +42,21 @@ class BroadcastDiscoveryService :
public RsBroadcastDiscovery, public RsTickingThread
{
public:
// TODO: std::shared_ptr<RsPeers> mRsPeers;
BroadcastDiscoveryService(RsPeers& pRsPeers);
virtual ~BroadcastDiscoveryService() override;
~BroadcastDiscoveryService() override;
/// @see RsBroadcastDiscovery
std::vector<RsBroadcastDiscoveryResult> getDiscoveredPeers() override;
/// @see RsBroadcastDiscovery
bool isMulticastListeningEnabled() override;
/// @see RsBroadcastDiscovery
bool enableMulticastListening() override;
/// @see RsBroadcastDiscovery
bool disableMulticastListening() override;
/// @see RsTickingThread
void data_tick() override;
@ -63,4 +76,22 @@ protected:
RsBroadcastDiscoveryResult createResult(
const UDC::IpPort& ipp, const std::string& uData );
#ifdef __ANDROID__
/** Android WifiManager.MulticastLock */
QAndroidJniObject mWifiMulticastLock;
/** Initialize the wifi multicast lock without acquiring it
* Needed to enable multicast listening in Android, for RetroShare broadcast
* discovery inspired by:
* https://github.com/flutter/flutter/issues/16335#issuecomment-420547860
*/
bool createMulticastLock();
/** Return false if mWifiMulticastLock is invalid and print error messages */
bool assertMulticastLockIsvalid();
#endif // def __ANDROID__
RS_SET_CONTEXT_DEBUG_LEVEL(3)
};

38
libretroshare/src/services/p3gxschannels.cc
View File

@ -44,9 +44,9 @@
#include "util/rsrandom.h"
#include "util/rsstring.h"
#ifdef RS_DEEP_SEARCH
# include "deep_search/deep_search.h"
#endif // RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
# include "deep_search/channelsindex.hpp"
#endif // RS_DEEP_CHANNEL_INDEX
/****
@ -1149,9 +1149,9 @@ bool p3GxsChannels::createChannelV2(
channelId = channel.mMeta.mGroupId;
#ifdef RS_DEEP_SEARCH
DeepSearch::indexChannelGroup(channel);
#endif // RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
DeepChannelsIndex::indexChannelGroup(channel);
#endif // RS_DEEP_CHANNEL_INDEX
return true;
}
@ -1180,9 +1180,9 @@ bool p3GxsChannels::createChannel(RsGxsChannelGroup& channel)
return false;
}
#ifdef RS_DEEP_SEARCH
DeepSearch::indexChannelGroup(channel);
#endif // RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
DeepChannelsIndex::indexChannelGroup(channel);
#endif // RS_DEEP_CHANNEL_INDEX
return true;
}
@ -1333,9 +1333,9 @@ bool p3GxsChannels::editChannel(RsGxsChannelGroup& channel)
return false;
}
#ifdef RS_DEEP_SEARCH
DeepSearch::indexChannelGroup(channel);
#endif // RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
DeepChannelsIndex::indexChannelGroup(channel);
#endif // RS_DEEP_CHANNEL_INDEX
return true;
}
@ -1401,9 +1401,9 @@ bool p3GxsChannels::createPostV2(
if(RsGenExchange::getPublishedMsgMeta(token,post.mMeta))
{
#ifdef RS_DEEP_SEARCH
DeepSearch::indexChannelPost(post);
#endif // RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
DeepChannelsIndex::indexChannelPost(post);
#endif // RS_DEEP_CHANNEL_INDEX
postId = post.mMeta.mMsgId;
return true;
@ -1787,9 +1787,9 @@ bool p3GxsChannels::createPost(RsGxsChannelPost& post)
if(RsGenExchange::getPublishedMsgMeta(token,post.mMeta))
{
#ifdef RS_DEEP_SEARCH
DeepSearch::indexChannelPost(post);
#endif // RS_DEEP_SEARCH
#ifdef RS_DEEP_CHANNEL_INDEX
DeepChannelsIndex::indexChannelPost(post);
#endif // RS_DEEP_CHANNEL_INDEX
return true;
}
@ -2466,4 +2466,6 @@ void p3GxsChannels::cleanTimedOutCallbacks()
} // RS_STACK_MUTEX(mDistantChannelsCallbacksMapMutex)
}
RsGxsChannelGroup::~RsGxsChannelGroup() = default;
RsGxsChannelPost::~RsGxsChannelPost() = default;
RsGxsChannels::~RsGxsChannels() = default;

12
libretroshare/src/services/p3idservice.cc
View File

@ -810,9 +810,15 @@ bool p3IdService::getOwnIds(std::list<RsGxsId> &ownIds,bool signed_only)
return true ;
}
bool p3IdService::isKnownId(const RsGxsId& id)
{
RS_STACK_MUTEX(mIdMtx);
return mKeyCache.is_cached(id) ||
std::find(mOwnIds.begin(), mOwnIds.end(),id) != mOwnIds.end();
}
bool p3IdService::identityToBase64( const RsGxsId& id,
std::string& base64String )
std::string& base64String )
{ return serialiseIdentityToMemory(id, base64String); }
bool p3IdService::serialiseIdentityToMemory( const RsGxsId& id,
@ -4788,3 +4794,7 @@ RsIdentityUsage::RsIdentityUsage(
RsIdentityUsage::RsIdentityUsage() :
mServiceId(RsServiceType::NONE), mUsageCode(UNKNOWN_USAGE), mAdditionalId(0)
{}
RsIdentity::~RsIdentity() = default;
RsReputationInfo::~RsReputationInfo() = default;
RsGixs::~RsGixs() = default;

10
libretroshare/src/services/p3idservice.h
View File

@ -293,12 +293,12 @@ public:
/// @see RsIdentity
bool getOwnPseudonimousIds(std::vector<RsGxsId>& ids) override;
virtual bool getOwnIds(std::list<RsGxsId> &ownIds, bool signed_only = false);
bool getOwnIds(
std::list<RsGxsId> &ownIds, bool signed_only = false ) override;
//virtual bool getPublicKey(const RsGxsId &id, RsTlvSecurityKey &key) ;
//virtual void networkRequestPublicKey(const RsGxsId& key_id,const std::list<RsPeerId>& peer_ids) ;
bool isKnownId(const RsGxsId& id) override;
virtual bool isOwnId(const RsGxsId& key_id) ;
bool isOwnId(const RsGxsId& key_id) override;
virtual bool signData( const uint8_t* data,
uint32_t data_size,
@ -619,7 +619,7 @@ private:
bool ownIdsAreLoaded() { RS_STACK_MUTEX(mIdMtx); return mOwnIdsLoaded; }
bool mAutoAddFriendsIdentitiesAsContacts;
uint32_t mMaxKeepKeysBanned ;
uint32_t mMaxKeepKeysBanned;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
};

340
libretroshare/src/services/p3msgservice.cc
View File

@ -3,7 +3,8 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -55,19 +56,10 @@
#include <map>
#include <sstream>
//#define MSG_DEBUG 1
//#define DEBUG_DISTANT_MSG
//#define DISABLE_DISTANT_MESSAGES
//#define DEBUG_DISTANT_MSG
typedef unsigned int uint;
using namespace Rs::Msgs;
static struct RsLog::logInfo msgservicezoneInfo = {RsLog::Default, "msgservice"};
#define msgservicezone &msgservicezoneInfo
static const uint32_t RS_MSG_DISTANT_MESSAGE_HASH_KEEP_TIME = 2*30*86400 ; // keep msg hashes for 2 months to avoid re-sent msgs
/// keep msg hashes for 2 months to avoid re-sent msgs
static constexpr uint32_t RS_MSG_DISTANT_MESSAGE_HASH_KEEP_TIME = 2*30*86400;
/* Another little hack ..... unique message Ids
* will be handled in this class.....
@ -128,11 +120,8 @@ uint32_t p3MsgService::getNewUniqueMsgId()
return mMsgUniqueId++;
}
int p3MsgService::tick()
int p3MsgService::tick()
{
pqioutput(PQL_DEBUG_BASIC, msgservicezone,
"p3MsgService::tick()");
/* don't worry about increasing tick rate!
* (handled by p3service)
*/
@ -158,7 +147,7 @@ void p3MsgService::cleanListOfReceivedMessageHashes()
{
RS_STACK_MUTEX(recentlyReceivedMutex);
rstime_t now = time(NULL);
rstime_t now = time(nullptr);
for( auto it = mRecentlyReceivedMessageHashes.begin();
it != mRecentlyReceivedMessageHashes.end(); )
@ -173,21 +162,13 @@ void p3MsgService::cleanListOfReceivedMessageHashes()
else ++it;
}
int p3MsgService::status()
{
pqioutput(PQL_DEBUG_BASIC, msgservicezone,
"p3MsgService::status()");
return 1;
}
void p3MsgService::processIncomingMsg(RsMsgItem *mi)
{
mi -> recvTime = time(NULL);
mi -> recvTime = static_cast<uint32_t>(time(nullptr));
mi -> msgId = getNewUniqueMsgId();
{
RsStackMutex stack(mMsgMtx); /*** STACK LOCKED MTX ***/
RS_STACK_MUTEX(mMsgMtx);
/* from a peer */
@ -226,6 +207,13 @@ void p3MsgService::processIncomingMsg(RsMsgItem *mi)
}
RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_ADD);
if(rsEvents)
{
std::shared_ptr<RsMailStatusEvent> pEvent(new RsMailStatusEvent());
pEvent->mChangedMsgIds.insert(std::to_string(mi->msgId));
rsEvents->postEvent(pEvent);
}
}
bool p3MsgService::checkAndRebuildPartialMessage(RsMsgItem *ci)
@ -289,7 +277,8 @@ void p3MsgService::handleIncomingItem(RsMsgItem *mi)
{
bool changed = false ;
if(checkAndRebuildPartialMessage(mi)) // only returns true when a msg is complete.
// only returns true when a msg is complete.
if(checkAndRebuildPartialMessage(mi))
{
processIncomingMsg(mi);
changed = true ;
@ -356,6 +345,9 @@ int p3MsgService::checkOutgoingMessages()
bool changed = false;
std::list<RsMsgItem*> output_queue;
using Evt_t = RsMailStatusEvent;
std::shared_ptr<Evt_t> pEvent(new Evt_t());
{
RS_STACK_MUTEX(mMsgMtx); /********** STACK LOCKED MTX ******/
@ -386,12 +378,12 @@ int p3MsgService::checkOutgoingMessages()
if(should_send)
{
/* send msg */
pqioutput( PQL_DEBUG_BASIC, msgservicezone,
"p3MsgService::checkOutGoingMessages() Sending out message");
Dbg3() << __PRETTY_FUNCTION__ << " Sending out message"
<< std::endl;
/* remove the pending flag */
output_queue.push_back(mit->second) ;
output_queue.push_back(mit->second);
pEvent->mChangedMsgIds.insert(std::to_string(mit->first));
/* When the message is a distant msg, dont remove it yet from
* the list. Only mark it as being sent, so that we don't send
@ -414,8 +406,8 @@ int p3MsgService::checkOutgoingMessages()
}
else
{
pqioutput( PQL_DEBUG_BASIC, msgservicezone,
"p3MsgService::checkOutGoingMessages() Delaying until available...");
Dbg3() << __PRETTY_FUNCTION__ << " Delaying until available..."
<< std::endl;
}
}
@ -446,6 +438,9 @@ int p3MsgService::checkOutgoingMessages()
if(changed)
RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_MOD);
if(rsEvents && !pEvent->mChangedMsgIds.empty())
rsEvents->postEvent(pEvent);
return 0;
}
@ -911,6 +906,8 @@ bool p3MsgService::removeMsgId(const std::string &mid)
}
bool changed = false;
using Evt_t = RsMailStatusEvent;
std::shared_ptr<Evt_t> pEvent(new Evt_t());
{
RsStackMutex stack(mMsgMtx); /********** STACK LOCKED MTX ******/
@ -922,6 +919,7 @@ bool p3MsgService::removeMsgId(const std::string &mid)
RsMsgItem *mi = mit->second;
imsg.erase(mit);
delete mi;
pEvent->mChangedMsgIds.insert(mid);
}
mit = msgOutgoing.find(msgId);
@ -931,6 +929,7 @@ bool p3MsgService::removeMsgId(const std::string &mid)
RsMsgItem *mi = mit->second;
msgOutgoing.erase(mit);
delete mi;
pEvent->mChangedMsgIds.insert(mid);
}
std::map<uint32_t, RsMsgSrcId*>::iterator srcIt = mSrcIds.find(msgId);
@ -938,6 +937,7 @@ bool p3MsgService::removeMsgId(const std::string &mid)
changed = true;
delete (srcIt->second);
mSrcIds.erase(srcIt);
pEvent->mChangedMsgIds.insert(mid);
}
}
@ -950,6 +950,9 @@ bool p3MsgService::removeMsgId(const std::string &mid)
RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_MOD);
}
if(rsEvents && !pEvent->mChangedMsgIds.empty())
rsEvents->postEvent(pEvent);
return changed;
}
@ -1095,12 +1098,14 @@ bool p3MsgService::setMsgParentId(uint32_t msgId, uint32_t msgParentId)
/****************************************/
/****************************************/
/* Message Items */
uint32_t p3MsgService::sendMessage(RsMsgItem *item) // no from field because it's implicitly our own PeerId
// no from field because it's implicitly our own PeerId
uint32_t p3MsgService::sendMessage(RsMsgItem* item)
{
if(!item)
return 0 ;
pqioutput(PQL_DEBUG_BASIC, msgservicezone, "p3MsgService::sendMessage()");
if(!item)
{
RsErr() << __PRETTY_FUNCTION__ << " item can't be null" << std::endl;
return 0;
}
item->msgId = getNewUniqueMsgId(); /* grabs Mtx as well */
item->msgFlags |= (RS_MSG_FLAGS_OUTGOING | RS_MSG_FLAGS_PENDING); /* add pending flag */
@ -1131,7 +1136,12 @@ uint32_t p3MsgService::sendMessage(RsMsgItem *item) // no from field because
uint32_t p3MsgService::sendDistantMessage(RsMsgItem *item, const RsGxsId& from)
{
if(!item) return 0;
if(!item)
{
RsErr() << __PRETTY_FUNCTION__ << " item can't be null" << std::endl;
print_stacktrace();
return 0;
}
item->msgId = getNewUniqueMsgId(); /* grabs Mtx as well */
item->msgFlags |= ( RS_MSG_FLAGS_DISTANT | RS_MSG_FLAGS_OUTGOING |
@ -1178,8 +1188,6 @@ bool p3MsgService::MessageSend(MessageInfo &info)
if (msg)
{
std::list<RsPgpId>::iterator it ;
if (msg->msgFlags & RS_MSG_FLAGS_SIGNED)
msg->msgFlags |= RS_MSG_FLAGS_SIGNATURE_CHECKS; // this is always true, since we are sending the message
@ -1192,14 +1200,125 @@ bool p3MsgService::MessageSend(MessageInfo &info)
imsg[msg->msgId] = msg;
RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_ADD);
//
// // return new message id
// rs_sprintf(info.msgId, "%lu", msg->msgId);
}
return true;
}
uint32_t p3MsgService::sendMail(
const RsGxsId from,
const std::string& subject,
const std::string& body,
const std::set<RsGxsId>& to,
const std::set<RsGxsId>& cc,
const std::set<RsGxsId>& bcc,
const std::vector<FileInfo>& attachments,
std::set<RsMailIdRecipientIdPair>& trackingIds,
std::string& errorMsg )
{
errorMsg.clear();
const std::string fname = __PRETTY_FUNCTION__;
auto pCheck = [&](bool test, const std::string& errMsg)
{
if(!test)
{
errorMsg = errMsg;
RsErr() << fname << " " << errMsg << std::endl;
}
return test;
};
if(!pCheck(!from.isNull(), "from can't be null")) return false;
if(!pCheck( rsIdentity->isOwnId(from),
"from must be own identity") ) return false;
if(!pCheck(!(to.empty() && cc.empty() && bcc.empty()),
"You must specify at least one recipient" )) return false;
auto dstCheck =
[&](const std::set<RsGxsId>& dstSet, const std::string& setName)
{
for(const RsGxsId& dst: dstSet)
{
if(dst.isNull())
{
errorMsg = setName + " contains a null recipient";
RsErr() << fname << " " << errorMsg << std::endl;
return false;
}
if(!rsIdentity->isKnownId(dst))
{
rsIdentity->requestIdentity(dst);
errorMsg = setName + " contains an unknown recipient: " +
dst.toStdString();
RsErr() << fname << " " << errorMsg << std::endl;
return false;
}
}
return true;
};
if(!dstCheck(to, "to")) return false;
if(!dstCheck(cc, "cc")) return false;
if(!dstCheck(bcc, "bcc")) return false;
MessageInfo msgInfo;
msgInfo.rsgxsid_srcId = from;
msgInfo.title = subject;
msgInfo.msg = body;
msgInfo.rsgxsid_msgto = to;
msgInfo.rsgxsid_msgcc = cc;
msgInfo.rsgxsid_msgbcc = bcc;
std::copy( attachments.begin(), attachments.end(),
std::back_inserter(msgInfo.files) );
uint32_t ret = 0;
using Evt_t = RsMailStatusEvent;
std::shared_ptr<Evt_t> pEvent(new Evt_t());
auto pSend = [&](const std::set<RsGxsId>& sDest)
{
for(const RsGxsId& dst : sDest)
{
RsMsgItem* msgItem = initMIRsMsg(msgInfo, dst);
if(!msgItem)
{
errorMsg += " initMIRsMsg from: " + from.toStdString()
+ " dst: " + dst.toStdString() + " subject: " + subject
+ " returned nullptr!\n";
RsErr() << fname << errorMsg;
continue;
}
uint32_t msgId = sendDistantMessage(msgItem, from);
// ensure we don't use that ptr again without noticing
msgItem = nullptr;
if(!msgId)
{
errorMsg += " sendDistantMessage from: " + from.toStdString()
+ " dst: " + dst.toStdString() + " subject: " + subject
+ " returned 0!\n";
RsErr() << fname << errorMsg;
continue;
}
const RsMailMessageId mailId = std::to_string(msgId);
pEvent->mChangedMsgIds.insert(mailId);
trackingIds.insert(RsMailIdRecipientIdPair(mailId, dst));
++ret;
}
};
pSend(to);
pSend(cc);
pSend(bcc);
if(rsEvents) rsEvents->postEvent(pEvent);
return ret;
}
bool p3MsgService::SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag)
{
if ((systemFlag & RS_MSG_SYSTEM) == 0) {
@ -1890,20 +2009,21 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id,
{
RS_STACK_MUTEX(mMsgMtx);
std::cerr << "(WW) p3MsgService::notifyDataStatus: Global router tells "
<< "us that item ID " << id
<< " could not be delivered on time.";
auto it = _ongoing_messages.find(id);
if(it == _ongoing_messages.end())
{
std::cerr << " (EE) cannot find pending message to acknowledge. "
<< "Weird. grouter id = " << id << std::endl;
RsErr() << __PRETTY_FUNCTION__
<< " cannot find pending message to acknowledge. "
<< "Weird. grouter id: " << id << std::endl;
return;
}
uint32_t msg_id = it->second;
std::cerr << " message id = " << msg_id << std::endl;
RsWarn() << __PRETTY_FUNCTION__ << " Global router tells "
<< "us that item ID " << id
<< " could not be delivered on time. Message id: "
<< msg_id << std::endl;
/* this is needed because it's not saved in config, but we should
* probably include it in _ongoing_messages */
@ -1912,10 +2032,10 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id,
std::map<uint32_t,RsMsgItem*>::iterator mit = msgOutgoing.find(msg_id);
if(mit == msgOutgoing.end())
{
std::cerr << " (II) message has been notified as not delivered, "
<< "but it's not in outgoing list. Probably it has been "
<< "delivered successfully by other means."
<< std::endl;
RsInfo() << __PRETTY_FUNCTION__
<< " message has been notified as not delivered, "
<< "but it's not in outgoing list. Probably it has been "
<< "delivered successfully by other means." << std::endl;
}
else
{
@ -1925,6 +2045,7 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id,
// clear the routed flag so that the message is requested again
mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED;
}
return;
}
@ -1954,17 +2075,31 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id,
return;
}
#if 0
delete it2->second;
msgOutgoing.erase(it2);
#else
// Do not delete it move to sent folder instead!
it2->second->msgFlags &= ~RS_MSG_FLAGS_PENDING;
imsg[msg_id] = it2->second;
msgOutgoing.erase(it2);
#endif
RsServer::notify()->notifyListChange( NOTIFY_LIST_MESSAGELIST,
NOTIFY_TYPE_ADD );
IndicateConfigChanged();
using Evt_t = RsMailStatusEvent;
std::shared_ptr<Evt_t> pEvent(new Evt_t());
pEvent->mChangedMsgIds.insert(std::to_string(msg_id));
if(rsEvents) rsEvents->postEvent(pEvent);
return;
}
std::cerr << "p3MsgService: unhandled data status info from global router"
<< " for msg ID " << id << ": this is a bug." << std::endl;
RsErr() << __PRETTY_FUNCTION__
<< " unhandled data status info from global router"
<< " for msg ID " << id << ": this is a bug." << std::endl;
}
bool p3MsgService::acceptDataFromPeer(const RsGxsId& to_gxs_id)
@ -1997,8 +2132,8 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId,
const RsGxsId& recipientId,
const uint8_t* data, uint32_t dataSize )
{
std::cout << __PRETTY_FUNCTION__ << " " << authorId << ", " << recipientId
<< ",, " << dataSize << std::endl;
Dbg2() << __PRETTY_FUNCTION__ << " " << authorId << ", " << recipientId
<< ",, " << dataSize << std::endl;
Sha1CheckSum hash = RsDirUtil::sha1sum(data, dataSize);
@ -2007,29 +2142,31 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId,
if( mRecentlyReceivedMessageHashes.find(hash) !=
mRecentlyReceivedMessageHashes.end() )
{
std::cerr << __PRETTY_FUNCTION__ << " (II) receiving "
RsInfo() << __PRETTY_FUNCTION__ << " (II) receiving "
<< "message of hash " << hash << " more than once. "
<< "Probably it has arrived before by other means."
<< std::endl;
return true;
}
mRecentlyReceivedMessageHashes[hash] = time(NULL);
mRecentlyReceivedMessageHashes[hash] =
static_cast<uint32_t>(time(nullptr));
}
IndicateConfigChanged();
RsItem *item = _serialiser->deserialise(const_cast<uint8_t*>(data), &dataSize);
RsItem *item = _serialiser->deserialise(
const_cast<uint8_t*>(data), &dataSize );
RsMsgItem *msg_item = dynamic_cast<RsMsgItem*>(item);
if(msg_item)
{
std::cerr << __PRETTY_FUNCTION__ << " Encrypted item correctly "
<< "deserialised. Passing on to incoming list."
<< std::endl;
Dbg3() << __PRETTY_FUNCTION__ << " Encrypted item correctly "
<< "deserialised. Passing on to incoming list."
<< std::endl;
msg_item->msgFlags |= RS_MSG_FLAGS_DISTANT;
/* we expect complete msgs - remove partial flag just in case
* someone has funny ideas */
* someone has funny ideas */
msg_item->msgFlags &= ~RS_MSG_FLAGS_PARTIAL;
// hack to pass on GXS id.
@ -2038,8 +2175,8 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId,
}
else
{
std::cerr << __PRETTY_FUNCTION__ << " Item could not be "
<< "deserialised. Format error??" << std::endl;
RsWarn() << __PRETTY_FUNCTION__ << " Item could not be "
<< "deserialised. Format error?" << std::endl;
return false;
}
@ -2049,8 +2186,11 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId,
bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId,
GxsTransSendStatus status )
{
std::cout << __PRETTY_FUNCTION__ << " " << mailId << ", "
<< static_cast<uint>(status) << std::endl;
Dbg2() << __PRETTY_FUNCTION__ << " " << mailId << ", "
<< static_cast<uint32_t>(status) << std::endl;
using Evt_t = RsMailStatusEvent;
std::shared_ptr<Evt_t> pEvent(new Evt_t());
if( status == GxsTransSendStatus::RECEIPT_RECEIVED )
{
@ -2062,11 +2202,10 @@ bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId,
auto it = gxsOngoingMessages.find(mailId);
if(it == gxsOngoingMessages.end())
{
std::cerr << __PRETTY_FUNCTION__<< " "
<< mailId
<< ", " << static_cast<uint>(status)
<< " (EE) cannot find pending message to acknowledge!"
<< std::endl;
RsErr() << __PRETTY_FUNCTION__<< " " << mailId << ", "
<< static_cast<uint32_t>(status)
<< " cannot find pending message to acknowledge!"
<< std::endl;
return false;
}
@ -2081,26 +2220,32 @@ bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId,
auto it2 = msgOutgoing.find(msg_id);
if(it2 == msgOutgoing.end())
{
std::cerr << __PRETTY_FUNCTION__ << " " << mailId
<< ", " << static_cast<uint>(status) << " (II) "
<< "received receipt for message that is not in "
<< "outgoing list, probably it has been acknoweldged "
<< "before by other means." << std::endl;
return true;
RsInfo() << __PRETTY_FUNCTION__ << " " << mailId
<< ", " << static_cast<uint32_t>(status)
<< " received receipt for message that is not in "
<< "outgoing list, probably it has been acknoweldged "
<< "before by other means." << std::endl;
}
else
{
#if 0
delete it2->second;
msgOutgoing.erase(it2);
#else
// Do not delete it move to sent folder instead!
it2->second->msgFlags &= ~RS_MSG_FLAGS_PENDING;
imsg[msg_id] = it2->second;
msgOutgoing.erase(it2);
#endif
pEvent->mChangedMsgIds.insert(std::to_string(msg_id));
}
delete it2->second;
msgOutgoing.erase(it2);
}
RsServer::notify()->notifyListChange( NOTIFY_LIST_MESSAGELIST,
NOTIFY_TYPE_ADD );
IndicateConfigChanged();
return true;
}
if( status >= GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE )
else if( status >= GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE )
{
uint32_t msg_id;
@ -2132,17 +2277,22 @@ bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId,
std::cerr << " message has been notified as not delivered, "
<< "but it not on outgoing list."
<< std::endl;
return true;
}
std::cerr << " reseting the ROUTED flag so that the message is "
<< "requested again" << std::endl;
else
{
std::cerr << " reseting the ROUTED flag so that the message is "
<< "requested again" << std::endl;
// clear the routed flag so that the message is requested again
mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED;
// clear the routed flag so that the message is requested again
mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED;
return true;
pEvent->mChangedMsgIds.insert(std::to_string(msg_id));
}
}
}
if(rsEvents && !pEvent->mChangedMsgIds.empty())
rsEvents->postEvent(pEvent);
return true;
}

19
libretroshare/src/services/p3msgservice.h
View File

@ -37,7 +37,7 @@
#include "services/p3service.h"
#include "rsitems/rsmsgitems.h"
#include "util/rsthreads.h"
#include "util/rsdebug.h"
#include "retroshare/rsgxsifacetypes.h"
#include "grouter/p3grouter.h"
@ -59,6 +59,19 @@ public:
virtual RsServiceInfo getServiceInfo();
/// @see RsMsgs::sendMail
uint32_t sendMail(const RsGxsId from,
const std::string& subject,
const std::string& body,
const std::set<RsGxsId>& to = std::set<RsGxsId>(),
const std::set<RsGxsId>& cc = std::set<RsGxsId>(),
const std::set<RsGxsId>& bcc = std::set<RsGxsId>(),
const std::vector<FileInfo>& attachments = std::vector<FileInfo>(),
std::set<RsMailIdRecipientIdPair>& trackingIds =
RS_DEFAULT_STORAGE_PARAM(std::set<RsMailIdRecipientIdPair>),
std::string& errorMsg =
RS_DEFAULT_STORAGE_PARAM(std::string) );
/* External Interface */
bool getMessageSummaries(std::list<Rs::Msgs::MsgInfoSummary> &msgList);
bool getMessage(const std::string &mid, Rs::Msgs::MessageInfo &msg);
@ -72,6 +85,7 @@ public:
// msgParentId == 0 --> remove
bool setMsgParentId(uint32_t msgId, uint32_t msgParentId);
RS_DEPRECATED_FOR(sendMail)
bool MessageSend(Rs::Msgs::MessageInfo &info);
bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag);
bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId);
@ -94,7 +108,6 @@ public:
//std::list<RsMsgItem *> &getMsgOutList();
int tick();
int status();
/*** Overloaded from p3Config ****/
virtual RsSerialiser *setupSerialiser();
@ -227,6 +240,8 @@ private:
bool mShouldEnableDistantMessaging ;
p3GxsTrans& mGxsTransServ;
RS_SET_CONTEXT_DEBUG_LEVEL(3)
};
#endif // MESSAGE_SERVICE_HEADER

65
libretroshare/src/turtle/p3turtle.cc
View File

@ -865,6 +865,8 @@ int p3turtle::handleIncoming()
//
void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
{
Dbg3() << __PRETTY_FUNCTION__ << " " << *item << std::endl;
// take a look at the item and test against inconsistent values
// - If the item destimation is
@ -877,11 +879,12 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
if(item_size > TURTLE_MAX_SEARCH_REQ_ACCEPTED_SERIAL_SIZE)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Dropping, because the serial size exceeds the accepted limit." << std::endl ;
#endif
std::cerr << " Caught a turtle search item with arbitrary large size from " << item->PeerId() << " of size " << item_size << " and depth " << item->depth << ". This is not allowed => dropping." << std::endl;
return ;
RsWarn() << __PRETTY_FUNCTION__
<< " Got a turtle search item with arbitrary large size from "
<< item->PeerId() << " of size " << item_size << " and depth "
<< item->depth << ". This is not allowed => dropping."
<< std::endl;
return;
}
{
@ -889,22 +892,20 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Dropping, because the search request cache is full." << std::endl ;
#endif
std::cerr << " More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl;
return ;
RsWarn() << __PRETTY_FUNCTION__ << " More than "
<< MAX_ALLOWED_SR_IN_CACHE << " search request in cache. "
<< "A peer is probably trying to flood your network See "
"the depth charts to find him." << std::endl;
return;
}
// If the item contains an already handled search request, give up. This
// happens when the same search request gets relayed by different peers
//
if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end())
if( _search_requests_origins.find(item->request_id) !=
_search_requests_origins.end() )
{
#ifdef P3TURTLE_DEBUG
std::cerr << " This is a bouncing request. Ignoring and deleting it." << std::endl ;
#endif
return ;
/* If the item contains an already handled search request, give up.
* This happens when the same search request gets relayed by
* different peers */
return;
}
}
@ -1013,13 +1014,21 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
// This function should be removed in the future, when file search will also use generic search items.
void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& search_results,uint32_t& max_allowed_hits)
void p3turtle::performLocalSearch(
RsTurtleSearchRequestItem *item, uint32_t& req_result_count,
std::list<RsTurtleSearchResultItem*>& search_results,
uint32_t& max_allowed_hits )
{
RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast<RsTurtleFileSearchRequestItem*>(item) ;
Dbg3() << __PRETTY_FUNCTION__ << " " << item << std::endl;
RsTurtleFileSearchRequestItem* ftsearch =
dynamic_cast<RsTurtleFileSearchRequestItem*>(item);
if(ftsearch != NULL)
{
performLocalSearch_files(ftsearch,req_result_count,search_results,max_allowed_hits) ;
performLocalSearch_files(
ftsearch, req_result_count, search_results,
max_allowed_hits );
return ;
}
@ -1060,12 +1069,13 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item
}
}
void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
void p3turtle::performLocalSearch_files(
RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count,
std::list<RsTurtleSearchResultItem*>& result,
uint32_t& max_allowed_hits )
{
#ifdef P3TURTLE_DEBUG
std::cerr << "Performing rsFiles->search()" << std::endl ;
#endif
// now, search!
Dbg3() << __PRETTY_FUNCTION__ << " " << *item << std::endl;
std::list<TurtleFileInfo> initialResults ;
item->search(initialResults) ;
@ -1104,6 +1114,9 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint
res_item = NULL ; // forces creation of a new item.
}
}
Dbg3() << __PRETTY_FUNCTION__ << " found " << req_result_count << " results"
<< std::endl;
}
void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)

9
libretroshare/src/turtle/p3turtle.h
View File

@ -19,6 +19,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#pragma once
//====================================== General setup of the router ===================================//
//
@ -130,10 +131,6 @@
// - should tunnels be re-used ? nope. The only useful case would be when two peers are exchanging files, which happens quite rarely.
//
#ifndef MRK_PQI_TURTLE_H
#define MRK_PQI_TURTLE_H
#include <string>
#include <list>
#include <set>
@ -464,6 +461,8 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
uint32_t _service_type ;
RS_SET_CONTEXT_DEBUG_LEVEL(1)
#ifdef P3TURTLE_DEBUG
// debug function
void dumpState() ;
@ -472,5 +471,3 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
void TS_dumpState();
#endif
};
#endif

14
libretroshare/src/use_libretroshare.pri
View File

@ -68,11 +68,23 @@ linux-* {
mLibs += dl
}
rs_deep_search {
rs_deep_channels_index | rs_deep_files_index {
mLibs += xapian
win32-g++:mLibs += rpcrt4
}
rs_deep_files_index_ogg {
mLibs += vorbisfile
}
rs_deep_files_index_flac {
mLibs += FLAC++
}
rs_deep_files_index_taglib {
mLibs += tag
}
rs_broadcast_discovery {
no_rs_cross_compiling {
UDP_DISCOVERY_SRC_PATH=$$clean_path($${RS_SRC_PATH}/supportlibs/udp-discovery-cpp/)

35
libretroshare/src/util/rsurl.cc
View File

@ -1,6 +1,6 @@
/*
* RetroShare
* Copyright (C) 2018 Gioacchino Mazzurco <gio@eigenlab.org>
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@ -17,15 +17,16 @@
*/
#include "rsurl.h"
#include "serialiser/rstypeserializer.h"
#include <cstdio>
#include <algorithm>
#include <iostream>
#include <iomanip>
#include <sstream>
#include "rsurl.h"
#include "serialiser/rstypeserializer.h"
#include "util/rsnet.h"
using namespace std;
RsUrl::RsUrl() : mPort(0), mHasPort(false) {}
@ -33,6 +34,26 @@ RsUrl::RsUrl() : mPort(0), mHasPort(false) {}
RsUrl::RsUrl(const std::string& urlStr) : mPort(0), mHasPort(false)
{ fromString(urlStr); }
RsUrl::RsUrl(const sockaddr_storage& addr): mPort(0), mHasPort(false)
{
switch(addr.ss_family)
{
case AF_INET: setScheme("ipv4"); break;
case AF_INET6: setScheme("ipv6"); break;
default:
{
std::string addrDump;
sockaddr_storage_dump(addr, &addrDump);
RsErr() << __PRETTY_FUNCTION__ << " got invalid addr: " << addrDump
<< std::endl;
return;
}
}
setHost(sockaddr_storage_iptostring(addr));
setPort(sockaddr_storage_port(addr));
}
RsUrl& RsUrl::fromString(const std::string& urlStr)
{
size_t endI = urlStr.size()-1;
@ -55,7 +76,7 @@ RsUrl& RsUrl::fromString(const std::string& urlStr)
{
if(++hostBeginI >= endI) return *this;
hostEndI = urlStr.find(ipv6WrapClose, hostBeginI);
mHost = urlStr.substr(hostBeginI, hostEndI - hostBeginI - 1);
mHost = urlStr.substr(hostBeginI, hostEndI - hostBeginI);
++hostEndI;
}
else
@ -250,10 +271,10 @@ RsUrl& RsUrl::setFragment(const std::string& fragment)
size_t boundary = len-2; // % Encoded char must be at least 2 hex char
for (size_t i = 0; i < len; ++i)
{
if(str[i] == '%' && i < boundary)
{
decoded << static_cast<char>(std::stoi(str.substr(++i, 2), 0, 16));
decoded << static_cast<char>(std::stoi(
str.substr(++i, 2), nullptr, 16 ));
++i;
}
else decoded << str[i];

5
libretroshare/src/util/rsurl.h
View File

@ -22,6 +22,8 @@
#include "serialiser/rsserializable.h"
struct sockaddr_storage;
/**
* Very simplistic and minimal URL helper class for RetroShare, after looking
* for a small and self-contained C/C++ URL parsing and manipulation library,
@ -36,7 +38,8 @@
struct RsUrl : RsSerializable
{
RsUrl();
RsUrl(const std::string& urlStr);
explicit RsUrl(const std::string& urlStr);
explicit RsUrl(const sockaddr_storage& ssas);
RsUrl& fromString(const std::string& urlStr);
std::string toString() const;

6
retroshare-gui/src/gui/ChatLobbyWidget.cpp
View File

@ -69,8 +69,8 @@
#define TYPE_LOBBY 1
#define IMAGE_CREATE ""
#define IMAGE_PUBLIC ":/images/chat_x24.png"
#define IMAGE_PRIVATE ":/images/chat_red24.png"
#define IMAGE_PUBLIC ":/icons/png/chats.png"
#define IMAGE_PRIVATE ":/icons/png/chats-private.png"
#define IMAGE_SUBSCRIBE ":/images/edit_add24.png"
#define IMAGE_UNSUBSCRIBE ":/images/cancel.png"
#define IMAGE_PEER_ENTERING ":images/user/add_user24.png"
@ -211,6 +211,8 @@ ChatLobbyWidget::ChatLobbyWidget(QWidget *parent, Qt::WindowFlags flags)
).arg(QString::number(2*S)).arg(QString::number(S)) ;
registerHelpButton(ui.helpButton,help_str,"ChatLobbyDialog") ;
ui.lobbyTreeWidget->setIconSize(QSize(S*1.5,S*1.5));
}
ChatLobbyWidget::~ChatLobbyWidget()

8
retroshare-gui/src/gui/ChatLobbyWidget.ui
View File

@ -228,12 +228,18 @@
</property>
<item row="0" column="0">
<widget class="QGroupBox" name="lobbyinfo_groupBox">
<property name="styleSheet">
<string notr="true"/>
</property>
<property name="title">
<string>Selected Chat Room info</string>
<string>Chat Room info</string>
</property>
<layout class="QVBoxLayout" name="lobbyinfo_groupBoxVLayout">
<item>
<layout class="QHBoxLayout" name="lobbyInfoHLayout">
<property name="topMargin">
<number>6</number>
</property>
<item>
<layout class="QVBoxLayout" name="lobbyInfoNameVLayout">
<item>

1
retroshare-gui/src/gui/chat/ChatWidget.cpp
View File

@ -113,6 +113,7 @@ ChatWidget::ChatWidget(QWidget *parent)
ui->searchButton->setIconSize(iconSize);
ui->sendButton->setFixedHeight(iconHeight);
ui->sendButton->setIconSize(iconSize);
ui->typingLabel->setMaximumHeight(QFontMetricsF(font()).height()*1.2);
//Initialize search
iCharToStartSearch=Settings->getChatSearchCharToStartSearch();

96
retroshare-gui/src/gui/chat/ChatWidget.ui
View File

@ -7,16 +7,25 @@
<x>0</x>
<y>0</y>
<width>667</width>
<height>334</height>
<height>528</height>
</rect>
</property>
<layout class="QGridLayout">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<number>0</number>
</property>
<property name="verticalSpacing">
<number>2</number>
</property>
<property name="margin">
<number>0</number>
</property>
<item row="1" column="0">
<layout class="QHBoxLayout" name="textChatHLayout">
<property name="spacing">
@ -37,7 +46,16 @@
<enum>QFrame::Raised</enum>
</property>
<layout class="QVBoxLayout" name="chatTextFrameVLayout">
<property name="margin">
<property name="leftMargin">
<number>0</number>
</property>
<property name="topMargin">
<number>0</number>
</property>
<property name="rightMargin">
<number>0</number>
</property>
<property name="bottomMargin">
<number>0</number>
</property>
<item>
@ -113,7 +131,16 @@
<enum>QFrame::Box</enum>
</property>
<layout class="QHBoxLayout" name="infoFrameHLayout">
<property name="margin">
<property name="leftMargin">
<number>6</number>
</property>
<property name="topMargin">
<number>6</number>
</property>
<property name="rightMargin">
<number>6</number>
</property>
<property name="bottomMargin">
<number>6</number>
</property>
<item>
@ -238,12 +265,6 @@ border-image: url(:/images/closepressed.png)
</item>
<item>
<widget class="QLabel" name="typingLabel">
<property name="maximumSize">
<size>
<width>400</width>
<height>18</height>
</size>
</property>
<property name="margin">
<number>0</number>
</property>
@ -291,7 +312,7 @@ border-image: url(:/images/closepressed.png)
<height>30</height>
</size>
</property>
<property name="placeholderText" stdset="0">
<property name="placeholderText">
<string>Type a message here</string>
</property>
</widget>
@ -311,7 +332,16 @@ border-image: url(:/images/closepressed.png)
<enum>QFrame::Sunken</enum>
</property>
<layout class="QHBoxLayout" name="toolBarFrameHLayout">
<property name="margin">
<property name="leftMargin">
<number>2</number>
</property>
<property name="topMargin">
<number>2</number>
</property>
<property name="rightMargin">
<number>2</number>
</property>
<property name="bottomMargin">
<number>2</number>
</property>
<item>
@ -520,7 +550,7 @@ border-image: url(:/images/closepressed.png)
<enum>Qt::NoFocus</enum>
</property>
<property name="icon">
<iconset resource="../images.qrc">
<iconset resource="../WikiPoos/Wiki_images.qrc">
<normaloff>:/images/arrow-left.png</normaloff>:/images/arrow-left.png</iconset>
</property>
<property name="autoRaise">
@ -546,7 +576,7 @@ border-image: url(:/images/closepressed.png)
<enum>Qt::NoFocus</enum>
</property>
<property name="icon">
<iconset resource="../images.qrc">
<iconset resource="../WikiPoos/Wiki_images.qrc">
<normaloff>:/images/arrow-right.png</normaloff>:/images/arrow-right.png</iconset>
</property>
<property name="autoRaise">
@ -583,7 +613,16 @@ border-image: url(:/images/closepressed.png)
<enum>QFrame::Plain</enum>
</property>
<layout class="QHBoxLayout" name="pluginButtonFrameHLayout">
<property name="margin">
<property name="leftMargin">
<number>2</number>
</property>
<property name="topMargin">
<number>2</number>
</property>
<property name="rightMargin">
<number>2</number>
</property>
<property name="bottomMargin">
<number>2</number>
</property>
</layout>
@ -675,7 +714,16 @@ border-image: url(:/images/closepressed.png)
<enum>QFrame::Sunken</enum>
</property>
<layout class="QHBoxLayout" name="titleBarFrameHLayout">
<property name="margin">
<property name="leftMargin">
<number>2</number>
</property>
<property name="topMargin">
<number>2</number>
</property>
<property name="rightMargin">
<number>2</number>
</property>
<property name="bottomMargin">
<number>2</number>
</property>
<item>
@ -756,7 +804,16 @@ border-image: url(:/images/closepressed.png)
<enum>QFrame::Plain</enum>
</property>
<layout class="QHBoxLayout" name="pluginTitleFrameHLayout">
<property name="margin">
<property name="leftMargin">
<number>2</number>
</property>
<property name="topMargin">
<number>2</number>
</property>
<property name="rightMargin">
<number>2</number>
</property>
<property name="bottomMargin">
<number>2</number>
</property>
</layout>
@ -998,6 +1055,7 @@ border-image: url(:/images/closepressed.png)
</customwidget>
</customwidgets>
<resources>
<include location="../WikiPoos/Wiki_images.qrc"/>
<include location="../images.qrc"/>
<include location="../icons.qrc"/>
</resources>

36
retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp
View File

@ -111,7 +111,7 @@ void PopupDistantChatDialog::updateDisplay()
switch(tinfo.status)
{
case RS_DISTANT_CHAT_STATUS_UNKNOWN:
//std::cerr << "Unknown hash. Error!" << std::endl;
_status_label->setIcon(QIcon(IMAGE_GRY_LED));
msg = tr("Remote status unknown.");
_status_label->setToolTip(msg);
@ -124,31 +124,31 @@ void PopupDistantChatDialog::updateDisplay()
case RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED:
std::cerr << "Chat remotely closed. " << std::endl;
_status_label->setIcon(QIcon(IMAGE_RED_LED));
_status_label->setToolTip(
QObject::tr("Distant peer has closed the chat") );
getChatWidget()->updateStatusString(
"%1", tr( "The person you are talking to has deleted the"
" secured chat tunnel." ), true );
getChatWidget()->blockSending(tr( "The chat partner deleted the secure"
" tunnel, messages will be delivered"
" as soon as possible"));
_status_label->setToolTip( QObject::tr("Distant peer has closed the chat") );
getChatWidget()->updateStatusString("%1", tr( "Your partner closed the conversation." ), true );
getChatWidget()->blockSending(tr( "Your partner closed the conversation."));
setPeerStatus(RS_STATUS_OFFLINE) ;
break ;
case RS_DISTANT_CHAT_STATUS_TUNNEL_DN:
//std::cerr << "Tunnel asked. Waiting for reponse. " << std::endl;
_status_label->setIcon(QIcon(IMAGE_YEL_LED));
msg = QObject::tr( "Tunnel is pending... Messages will be delivered as"
" soon as possible" );
msg = QObject::tr( "Tunnel is pending");
if(tinfo.pending_items > 0)
msg += QObject::tr("(some undelivered messages)") ; // we cannot use the pending_items count because it accounts for ACKS and keep alive packets as well.
_status_label->setToolTip(msg);
getChatWidget()->updateStatusString("%1", msg, true);
getChatWidget()->blockSending(msg);
setPeerStatus(RS_STATUS_OFFLINE);
break;
case RS_DISTANT_CHAT_STATUS_CAN_TALK:
//std::cerr << "Tunnel is ok and data is transmitted." << std::endl;
_status_label->setIcon(QIcon(IMAGE_GRN_LED));
msg = QObject::tr( "Secured tunnel is working. "
"Messages are delivered immediately!" );
msg = QObject::tr( "End-to-end encrypted conversation established");
_status_label->setToolTip(msg);
getChatWidget()->unblockSending();
setPeerStatus(RS_STATUS_ONLINE);
@ -158,17 +158,15 @@ void PopupDistantChatDialog::updateDisplay()
void PopupDistantChatDialog::closeEvent(QCloseEvent *e)
{
//std::cerr << "Closing window => closing distant chat for hash " << _pid << std::endl;
DistantChatPeerInfo tinfo ;
rsMsgs->getDistantChatStatus(_tunnel_id,tinfo) ;
if(tinfo.status != RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED)
{
QString msg = tr("Closing this window will end the conversation, notify the peer and remove the encrypted tunnel.") ;
QString msg = tr("Closing this window will end the conversation. Unsent messages will be dropped.") ;
if(QMessageBox::Ok == QMessageBox::critical(NULL,tr("Kill the tunnel?"),msg, QMessageBox::Ok | QMessageBox::Cancel))
if(QMessageBox::Ok == QMessageBox::critical(NULL,tr("Close conversation?"),msg, QMessageBox::Ok | QMessageBox::Cancel))
rsMsgs->closeDistantChatConnexion(_tunnel_id) ;
else
{

1
retroshare-gui/src/gui/icons.qrc
View File

@ -271,6 +271,7 @@
<file>icons/png/correct.png</file>
<file>icons/png/comment.png</file>
<file>icons/png/chats.png</file>
<file>icons/png/chats-private.png</file>
<file>icons/png/fileshare.png</file>
<file>icons/png/forum.png</file>
<file>icons/png/message.png</file>

BIN
retroshare-gui/src/gui/icons/png/chats-private.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 3.6 KiB

64
retroshare-gui/src/gui/icons/svg/chats-private.svg Normal file
View File

@ -0,0 +1,64 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!-- Created with Inkscape (http://www.inkscape.org/) -->
<svg
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:cc="http://creativecommons.org/ns#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:svg="http://www.w3.org/2000/svg"
xmlns="http://www.w3.org/2000/svg"
xmlns:sodipodi="http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd"
xmlns:inkscape="http://www.inkscape.org/namespaces/inkscape"
id="svg4155"
version="1.1"
inkscape:version="0.92.3 (2405546, 2018-03-11)"
xml:space="preserve"
width="80"
height="80"
viewBox="0 0 80 80"
sodipodi:docname="chats-private.svg"
inkscape:export-filename="C:\Users\Mustermann\Documents\GitHub\RetroShare\retroshare-gui\src\gui\icons\png\chats-private.png"
inkscape:export-xdpi="153"
inkscape:export-ydpi="153"><metadata
id="metadata4161"><rdf:RDF><cc:Work
rdf:about=""><dc:format>image/svg+xml</dc:format><dc:type
rdf:resource="http://purl.org/dc/dcmitype/StillImage" /><dc:title></dc:title></cc:Work></rdf:RDF></metadata><defs
id="defs4159" /><sodipodi:namedview
pagecolor="#ffffff"
bordercolor="#666666"
borderopacity="1"
objecttolerance="10"
gridtolerance="10"
guidetolerance="10"
inkscape:pageopacity="0"
inkscape:pageshadow="2"
inkscape:window-width="1152"
inkscape:window-height="801"
id="namedview4157"
showgrid="false"
inkscape:zoom="5.11875"
inkscape:cx="27.545788"
inkscape:cy="55.377339"
inkscape:window-x="-8"
inkscape:window-y="-8"
inkscape:window-maximized="1"
inkscape:current-layer="g4163"><inkscape:grid
type="xygrid"
id="grid4146" /></sodipodi:namedview><g
id="g4163"
inkscape:groupmode="layer"
inkscape:label="ink_ext_XXXXXX"
transform="matrix(1.25,0,0,-1.25,0,80)"><g
transform="matrix(2.0115028,0,0,-2.0115028,1.8333347,61.674992)"
id="g3"
style="opacity:0.97000002;fill:#00b400;fill-opacity:1"><g
id="g5"
style="fill:#00b400;fill-opacity:1"><path
style="fill:#00b400;fill-opacity:1"
inkscape:connector-curvature="0"
d="m 30,18.3 c 0,-2.682 -1.877,-5.024 -4.673,-6.291 0.164,0.585 0.259,1.189 0.276,1.809 0.279,0.166 0.549,0.345 0.801,0.54 1.41,1.088 2.188,2.487 2.188,3.942 0,1.454 -0.775,2.854 -2.188,3.941 -1.547,1.192 -3.627,1.85 -5.855,1.85 -1.504,0 -2.938,-0.301 -4.188,-0.859 -0.778,0.153 -1.585,0.25 -2.416,0.284 1.705,1.282 4.033,2.073 6.603,2.073 1.648,0 3.198,-0.325 4.549,-0.898 l 3.439,1.756 -0.695,-3.511 C 29.191,21.676 30,20.061 30,18.3 Z"
id="path7" /><path
style="fill:#00b400;fill-opacity:1"
inkscape:connector-curvature="0"
d="M 24.625,12.837 C 24.607,12.218 24.512,11.613 24.348,11.029 23.147,6.755 18.215,3.554 12.314,3.554 5.513,3.554 0,7.806 0,13.051 c 0,2.295 1.055,4.399 2.812,6.041 l -0.906,4.573 4.483,-2.287 c 1.758,0.746 3.777,1.17 5.925,1.17 0.219,0 0.437,-0.004 0.653,-0.014 0.829,-0.032 1.637,-0.129 2.414,-0.283 5.316,-1.051 9.246,-4.771 9.246,-9.2 0,-0.072 -10e-4,-0.143 -0.002,-0.214 z M 6.506,14.615 c -0.99,0 -1.792,-0.802 -1.792,-1.791 0,-0.99 0.802,-1.792 1.792,-1.792 0.989,0 1.792,0.802 1.792,1.792 0,0.988 -0.802,1.791 -1.792,1.791 z m 5.808,0 c -0.99,0 -1.792,-0.802 -1.792,-1.791 0,-0.99 0.802,-1.792 1.792,-1.792 0.989,0 1.792,0.802 1.792,1.792 0,0.988 -0.803,1.791 -1.792,1.791 z m 5.808,0 c -0.989,0 -1.792,-0.802 -1.792,-1.791 0,-0.99 0.803,-1.792 1.792,-1.792 0.989,0 1.792,0.802 1.792,1.792 0,0.988 -0.803,1.791 -1.792,1.791 z"
id="path9" /></g></g></g></svg>
Side by Side

After

Width:  |  Height:  |  Size: 3.6 KiB

5
retroshare-gui/src/gui/qss/stylesheet/Standard.qss
View File

@ -176,8 +176,9 @@ ChatLobbyWidget QGroupBox#lobbyinfo_groupBox
ChatLobbyWidget QGroupBox::title#lobbyinfo_groupBox
{
padding: 4 12px;
background: transparent;
padding: 4 12px;
background: #039bd5;
color: white;
}
ChatLobbyWidget QLabel#lobbyInfoLabel {

4
retroshare-service/src/android/AndroidManifest.xml
View File

@ -85,4 +85,8 @@
<!-- Added by G10h4ck: Needed permission for network usage -->
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
<!-- Added by G10h4ck: Needed to listen for multicast packets, needed for
! broadcast discovery -->
<uses-permission android:name="android.permission.ACCESS_WIFI_STATE" />
<uses-permission android:name="android.permission.CHANGE_WIFI_MULTICAST_STATE" />
</manifest>

49
retroshare.pri
View File

@ -159,16 +159,37 @@ rs_macos10.10:CONFIG -= rs_macos10.11
rs_macos10.12:CONFIG -= rs_macos10.11
rs_macos10.13:CONFIG -= rs_macos10.11
rs_macos10.14:CONFIG -= rs_macos10.11
rs_macos10.15:CONFIG -= rs_macos10.11
# To enable JSON API append the following assignation to qmake command line
# "CONFIG+=rs_jsonapi"
CONFIG *= no_rs_jsonapi
rs_jsonapi:CONFIG -= no_rs_jsonapi
# To enable deep search append the following assignation to qmake command line
# CONFIG *= rs_deep_search
CONFIG *= no_rs_deep_search
rs_deep_search:CONFIG -= no_rs_deep_search
# To enable channel indexing append the following assignation to qmake command
# line "CONFIG+=rs_deep_channel_index"
CONFIG *= no_rs_deep_channel_index
rs_deep_channel_index:CONFIG -= no_rs_deep_channel_index
# To enable deep files indexing append the following assignation to qmake
# command line "CONFIG+=rs_files_index"
CONFIG *= no_rs_deep_files_index
rs_deep_files_index:CONFIG -= no_rs_deep_files_index
# To enable Ogg files deep indexing append the following assignation to qmake
# command line "CONFIG+=rs_deep_files_index_ogg"
CONFIG *= no_rs_deep_files_index_ogg
rs_deep_files_index_ogg:CONFIG -= no_rs_deep_files_index_ogg
# To enable FLAC files deep indexing append the following assignation to qmake
# command line "CONFIG+=rs_deep_files_index_flac"
CONFIG *= no_rs_deep_files_index_flac
rs_deep_files_index_flac:CONFIG -= no_rs_deep_files_index_flac
# To enable taglib files deep indexing append the following assignation to qmake
# command line "CONFIG+=rs_deep_files_index_taglib"
CONFIG *= no_rs_deep_files_index_taglib
rs_deep_files_index_taglib:CONFIG -= no_rs_deep_files_index_taglib
# To enable native dialogs append the following assignation to qmake command
# line "CONFIG+=rs_use_native_dialogs"
@ -564,15 +585,12 @@ retroshare_qml_app {
warning("QMAKE: you have enabled retroshare_qml_app which is deprecated")
}
rs_deep_search {
DEFINES *= RS_DEEP_SEARCH
rs_deep_channels_index:DEFINES *= RS_DEEP_CHANNEL_INDEX
linux {
exists("/usr/include/xapian-1.3") {
INCLUDEPATH += /usr/include/xapian-1.3
}
}
}
rs_deep_files_index:DEFINES *= RS_DEEP_FILES_INDEX
rs_deep_files_index_ogg:DEFINES *= RS_DEEP_FILES_INDEX_OGG
rs_deep_files_index_flac:DEFINES *= RS_DEEP_FILES_INDEX_FLAC
rs_deep_files_index_taglib:DEFINES *= RS_DEEP_FILES_INDEX_TAGLIB
rs_use_native_dialogs:DEFINES *= RS_NATIVEDIALOGS
@ -743,6 +761,13 @@ macx-* {
QMAKE_CXXFLAGS += -Wno-nullability-completeness
QMAKE_CFLAGS += -Wno-nullability-completeness
}
rs_macos10.15 {
message(***retroshare.pri: Set Target and SDK to MacOS 10.15 )
QMAKE_MACOSX_DEPLOYMENT_TARGET=10.15
QMAKE_MAC_SDK = macosx10.15
QMAKE_CXXFLAGS += -Wno-nullability-completeness
QMAKE_CFLAGS += -Wno-nullability-completeness
}