diff --git a/build_scripts/Android/prepare-toolchain-clang.sh b/build_scripts/Android/prepare-toolchain-clang.sh index b943ee5e3..e01b40f55 100755 --- a/build_scripts/Android/prepare-toolchain-clang.sh +++ b/build_scripts/Android/prepare-toolchain-clang.sh @@ -424,7 +424,7 @@ build_restbed() rm -rf restbed-build; mkdir restbed-build ; cd restbed-build cmake \ - -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON BUILD_TESTS=OFF \ -DBUILD_SSL=OFF -DCMAKE_INSTALL_PREFIX="${PREFIX}" -B. -H../restbed make -j${HOST_NUM_CPU} make install diff --git a/build_scripts/Debian/debian/copyright b/build_scripts/Debian/debian/copyright index c94948a36..fe0eb5010 100644 --- a/build_scripts/Debian/debian/copyright +++ b/build_scripts/Debian/debian/copyright @@ -1,19 +1,23 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: retroshare -Upstream-Contact: retroshare.team@gmail.com +Upstream-Contact: contact@retroshare.cc Source: https://github.com/retroshare/retroshare Files: openpgpsdk/* -Copyright: 2005-2008 Ben Laurie, Rachel Willmer, Retroshare Team +Copyright: 2005-2008 Ben Laurie, Rachel Willmer, Retroshare Team License: Apache-2.0 Files: jsonapi-generator/* libretroshare/src/jsonapi/* Copyright: 2018-2019 Gioacchino Mazzurco -License: AGPL-3+ +License: AGPL-3.0-or-later + +Files: libretroshare/src/deep_search/* +Copyright: 2018-2019 Gioacchino Mazzurco +License: AGPL-3.0-only Files: libretroshare/* -Copyright: 2007-2018, Retroshare Team -License: LGPL-3+ +Copyright: 2007-2019, Retroshare Team +License: LGPL-3.0-or-later Files: src/retroshare-gui/src/TorControl/ Copyright: 2014, John Brooks @@ -28,8 +32,8 @@ Copyright: 2013 Jeff Weinstein License: MIT Files: * -Copyright: 2007-2018, Retroshare Team -License: AGPL-3+ +Copyright: 2007-2019, Retroshare Team +License: AGPL-3.0-only ####### # TODO @@ -56,7 +60,7 @@ License: Apache-2.0 See the License for the specific language governing permissions and limitations under the License. -License: LGPL-3+ +License: LGPL-3.0-or-later This program is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 3 of the @@ -75,7 +79,7 @@ License: LGPL-3+ OpenSSL that use the same license as OpenSSL), and distribute linked combinations including the two. -License: AGPL-3+ +License: AGPL-3.0-or-later This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the @@ -86,11 +90,36 @@ License: AGPL-3+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. . - You should have received a copy of the GNU Lesser General Public License - along with this program. If not, see . + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . . As a special exception, the copyright holders give permission to link the - code of portions of this program with the OpenSSL library under certain + code or portions of this program with the OpenSSL library under certain + conditions as described in each individual source file and distribute + linked combinations including the program with the OpenSSL library. You + must comply with the GNU Affero General Public License in all respects for + all of the code used other than as permitted herein. If you modify file(s) + with this exception, you may extend this exception to your version of the + file(s), but you are not obligated to do so. If you do not wish to do so, + delete this exception statement from your version. If you delete this + exception statement from all source files in the program, then also delete + it in the license file. + +License: AGPL-3.0-only + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License version 3 as + published by the Free Software Foundation. + . + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + . + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + . + As a special exception, the copyright holders give permission to link the + code or portions of this program with the OpenSSL library under certain conditions as described in each individual source file and distribute linked combinations including the program with the OpenSSL library. You must comply with the GNU Affero General Public License in all respects for diff --git a/libretroshare/src/chat/distantchat.cc b/libretroshare/src/chat/distantchat.cc index cc372b01c..621e92229 100644 --- a/libretroshare/src/chat/distantchat.cc +++ b/libretroshare/src/chat/distantchat.cc @@ -178,8 +178,7 @@ bool DistantChatService::acceptDataFromPeer(const RsGxsId& gxs_id,const RsGxsTun return res ; } -void DistantChatService::notifyTunnelStatus( - const RsGxsTunnelId& tunnel_id, uint32_t tunnel_status ) +void DistantChatService::notifyTunnelStatus( const RsGxsTunnelId& tunnel_id, uint32_t tunnel_status ) { #ifdef DEBUG_DISTANT_CHAT DISTANT_CHAT_DEBUG() << "DistantChatService::notifyTunnelStatus(): got notification " << std::hex << tunnel_status << std::dec << " for tunnel " << tunnel_id << std::endl; @@ -195,18 +194,17 @@ void DistantChatService::notifyTunnelStatus( RsServer::notify()->notifyPeerStatusChanged(tunnel_id.toStdString(),RS_STATUS_ONLINE) ; break ; - case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"tunnel is down...") ; + case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"Tunnel is down...") ; RsServer::notify()->notifyPeerStatusChanged(tunnel_id.toStdString(),RS_STATUS_OFFLINE) ; break ; - case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"tunnel is down...") ; + case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED: RsServer::notify()->notifyChatStatus(ChatId(DistantChatPeerId(tunnel_id)),"Tunnel is down...") ; RsServer::notify()->notifyPeerStatusChanged(tunnel_id.toStdString(),RS_STATUS_OFFLINE) ; break ; } } -void DistantChatService::receiveData( - const RsGxsTunnelId& tunnel_id, unsigned char* data, uint32_t data_size) +void DistantChatService::receiveData( const RsGxsTunnelId& tunnel_id, unsigned char* data, uint32_t data_size) { #ifdef DEBUG_DISTANT_CHAT DISTANT_CHAT_DEBUG() << "DistantChatService::receiveData(): got data of size " << std::dec << data_size << " for tunnel " << tunnel_id << std::endl; @@ -303,16 +301,14 @@ bool DistantChatService::getDistantChatStatus(const DistantChatPeerId& tunnel_id cinfo.to_id = tinfo.destination_gxs_id; cinfo.own_id = tinfo.source_gxs_id; + cinfo.pending_items = tinfo.pending_data_packets; cinfo.peer_id = tunnel_id; switch(tinfo.tunnel_status) { - case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_CAN_TALK : - cinfo.status = RS_DISTANT_CHAT_STATUS_CAN_TALK; break; - case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN: - cinfo.status = RS_DISTANT_CHAT_STATUS_TUNNEL_DN; break; - case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED: - cinfo.status = RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED; break; + case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_CAN_TALK : cinfo.status = RS_DISTANT_CHAT_STATUS_CAN_TALK; break; + case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_TUNNEL_DN: cinfo.status = RS_DISTANT_CHAT_STATUS_TUNNEL_DN; break; + case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_REMOTELY_CLOSED: cinfo.status = RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED; break; case RsGxsTunnelService::RS_GXS_TUNNEL_STATUS_UNKNOWN: default: cinfo.status = RS_DISTANT_CHAT_STATUS_UNKNOWN; break; diff --git a/libretroshare/src/deep_search/channelsindex.cpp b/libretroshare/src/deep_search/channelsindex.cpp new file mode 100644 index 000000000..cd1c374fc --- /dev/null +++ b/libretroshare/src/deep_search/channelsindex.cpp @@ -0,0 +1,230 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "deep_search/channelsindex.hpp" +#include "deep_search/commonutils.hpp" + +uint32_t DeepChannelsIndex::search( + const std::string& queryStr, + std::vector& results, uint32_t maxResults ) +{ + results.clear(); + + std::unique_ptr dbPtr( + DeepSearch::openReadOnlyDatabase(dbPath()) ); + if(!dbPtr) return 0; + + Xapian::Database& db(*dbPtr); + + // Set up a QueryParser with a stemmer and suitable prefixes. + Xapian::QueryParser queryparser; + //queryparser.set_stemmer(Xapian::Stem("en")); + queryparser.set_stemming_strategy(queryparser.STEM_SOME); + // Start of prefix configuration. + //queryparser.add_prefix("title", "S"); + //queryparser.add_prefix("description", "XD"); + // End of prefix configuration. + + // And parse the query. + Xapian::Query query = queryparser.parse_query(queryStr); + + // Use an Enquire object on the database to run the query. + Xapian::Enquire enquire(db); + enquire.set_query(query); + + Xapian::MSet mset = enquire.get_mset( + 0, maxResults ? maxResults : db.get_doccount() ); + + for ( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m ) + { + const Xapian::Document& doc = m.get_document(); + DeepChannelsSearchResult s; + s.mUrl = doc.get_value(URL_VALUENO); +#if XAPIAN_AT_LEAST(1,3,5) + s.mSnippet = mset.snippet(doc.get_data()); +#endif // XAPIAN_AT_LEAST(1,3,5) + results.push_back(s); + } + + return static_cast(results.size()); +} + +void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan) +{ + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); + + // Set up a TermGenerator that we'll use in indexing. + Xapian::TermGenerator termgenerator; + //termgenerator.set_stemmer(Xapian::Stem("en")); + + // We make a document and tell the term generator to use this. + Xapian::Document doc; + termgenerator.set_document(doc); + + // Index each field with a suitable prefix. + termgenerator.index_text(chan.mMeta.mGroupName, 1, "G"); + termgenerator.index_text( + DeepSearch::timetToXapianDate(chan.mMeta.mPublishTs), 1, "D" ); + termgenerator.index_text(chan.mDescription, 1, "XD"); + + // Index fields without prefixes for general search. + termgenerator.index_text(chan.mMeta.mGroupName); + termgenerator.increase_termpos(); + termgenerator.index_text(chan.mDescription); + + RsUrl chanUrl; chanUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", chan.mMeta.mGroupId.toStdString()); + const std::string idTerm("Q" + chanUrl.toString()); + + chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs)); + chanUrl.setQueryKV("name", chan.mMeta.mGroupName); + if(!chan.mMeta.mAuthorId.isNull()) + chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString()); + if(chan.mMeta.mSignFlags) + chanUrl.setQueryKV( "signFlags", + std::to_string(chan.mMeta.mSignFlags) ); + std::string rsLink(chanUrl.toString()); + + // store the RS link so we are able to retrive it on matching search + doc.add_value(URL_VALUENO, rsLink); + + // Store some fields for display purposes. + doc.set_data(chan.mMeta.mGroupName + "\n" + chan.mDescription); + + // We use the identifier to ensure each object ends up in the + // database only once no matter how many times we run the + // indexer. "Q" prefix is a Xapian convention for unique id term. + doc.add_boolean_term(idTerm); + db.replace_document(idTerm, doc); +} + +void DeepChannelsIndex::removeChannelFromIndex(RsGxsGroupId grpId) +{ + // "Q" prefix is a Xapian convention for unique id term. + RsUrl chanUrl; chanUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", grpId.toStdString()); + std::string idTerm("Q" + chanUrl.toString()); + + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); + db.delete_document(idTerm); +} + +void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post) +{ + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); + + // Set up a TermGenerator that we'll use in indexing. + Xapian::TermGenerator termgenerator; + //termgenerator.set_stemmer(Xapian::Stem("en")); + + // We make a document and tell the term generator to use this. + Xapian::Document doc; + termgenerator.set_document(doc); + + // Index each field with a suitable prefix. + termgenerator.index_text(post.mMeta.mMsgName, 1, "S"); + termgenerator.index_text( + DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" ); + + // TODO: we should strip out HTML tags instead of skipping indexing + // Avoid indexing HTML + bool isPlainMsg = + post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>'; + + if(isPlainMsg) + termgenerator.index_text(post.mMsg, 1, "XD"); + + // Index fields without prefixes for general search. + termgenerator.index_text(post.mMeta.mMsgName); + if(isPlainMsg) + { + termgenerator.increase_termpos(); + termgenerator.index_text(post.mMsg); + } + + for(const RsGxsFile& attachment : post.mFiles) + { + termgenerator.index_text(attachment.mName, 1, "F"); + + termgenerator.increase_termpos(); + termgenerator.index_text(attachment.mName); + } + + // We use the identifier to ensure each object ends up in the + // database only once no matter how many times we run the + // indexer. + RsUrl postUrl; postUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", post.mMeta.mGroupId.toStdString()) + .setQueryKV("msgid", post.mMeta.mMsgId.toStdString()); + std::string idTerm("Q" + postUrl.toString()); + + postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs)); + postUrl.setQueryKV("name", post.mMeta.mMsgName); + postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString()); + std::string rsLink(postUrl.toString()); + + // store the RS link so we are able to retrive it on matching search + doc.add_value(URL_VALUENO, rsLink); + + // Store some fields for display purposes. + if(isPlainMsg) + doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg); + else doc.set_data(post.mMeta.mMsgName); + + doc.add_boolean_term(idTerm); + db.replace_document(idTerm, doc); +} + +void DeepChannelsIndex::removeChannelPostFromIndex( + RsGxsGroupId grpId, RsGxsMessageId msgId ) +{ + RsUrl postUrl; postUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", grpId.toStdString()) + .setQueryKV("msgid", msgId.toStdString()); + // "Q" prefix is a Xapian convention for unique id term. + std::string idTerm("Q" + postUrl.toString()); + + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); + db.delete_document(idTerm); +} diff --git a/libretroshare/src/deep_search/channelsindex.hpp b/libretroshare/src/deep_search/channelsindex.hpp new file mode 100644 index 000000000..0a49629d9 --- /dev/null +++ b/libretroshare/src/deep_search/channelsindex.hpp @@ -0,0 +1,77 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +#include +#include + +#include "util/rstime.h" +#include "retroshare/rsgxschannels.h" +#include "retroshare/rsinit.h" +#include "util/rsurl.h" + +struct DeepChannelsSearchResult +{ + std::string mUrl; + double mWeight; + std::string mSnippet; +}; + +struct DeepChannelsIndex +{ + /** + * @brief Search indexed GXS groups and messages + * @param[in] maxResults maximum number of acceptable search results, 0 for + * no limits + * @return search results count + */ + static uint32_t search( const std::string& queryStr, + std::vector& results, + uint32_t maxResults = 100 ); + + static void indexChannelGroup(const RsGxsChannelGroup& chan); + + static void removeChannelFromIndex(RsGxsGroupId grpId); + + static void indexChannelPost(const RsGxsChannelPost& post); + + static void removeChannelPostFromIndex( + RsGxsGroupId grpId, RsGxsMessageId msgId ); + + static uint32_t indexFile(const std::string& path); + +private: + + enum : Xapian::valueno + { + /// Used to store retroshare url of indexed documents + URL_VALUENO, + + /// @see Xapian::BAD_VALUENO + BAD_VALUENO = Xapian::BAD_VALUENO + }; + + static const std::string& dbPath() + { + static const std::string dbDir = + RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; + return dbDir; + } +}; diff --git a/libretroshare/src/deep_search/commonutils.cpp b/libretroshare/src/deep_search/commonutils.cpp new file mode 100644 index 000000000..eecbd4ec6 --- /dev/null +++ b/libretroshare/src/deep_search/commonutils.cpp @@ -0,0 +1,93 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "deep_search/commonutils.hpp" +#include "util/stacktrace.h" +#include "util/rsdebug.h" + +namespace DeepSearch +{ + +std::unique_ptr openWritableDatabase( + const std::string& path, int flags, int blockSize ) +{ + try + { + std::unique_ptr dbPtr( + new Xapian::WritableDatabase(path, flags, blockSize) ); + return dbPtr; + } + catch(Xapian::DatabaseLockError) + { + RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock " + << path << std::endl; + print_stacktrace(); + } + catch(...) + { + RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted " + << "deleting it might help without causing any harm: " + << path << std::endl; + print_stacktrace(); + } + + return nullptr; +} + +std::unique_ptr openReadOnlyDatabase( + const std::string& path, int flags ) +{ + try + { + std::unique_ptr dbPtr( + new Xapian::Database(path, flags) ); + return dbPtr; + } + catch(Xapian::DatabaseOpeningError e) + { + RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg() + << ", probably nothing has been indexed yet." << std::endl; + } + catch(Xapian::DatabaseLockError) + { + RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock " + << path << std::endl; + print_stacktrace(); + } + catch(...) + { + RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted " + << "deleting it might help without causing any harm: " + << path << std::endl; + print_stacktrace(); + } + + return nullptr; +} + +std::string timetToXapianDate(const rstime_t& time) +{ + char date[] = "YYYYMMDD\0"; + time_t tTime = static_cast(time); + std::strftime(date, 9, "%Y%m%d", std::gmtime(&tTime)); + return date; +} + +} diff --git a/libretroshare/src/deep_search/commonutils.hpp b/libretroshare/src/deep_search/commonutils.hpp new file mode 100644 index 000000000..28961bc09 --- /dev/null +++ b/libretroshare/src/deep_search/commonutils.hpp @@ -0,0 +1,45 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +#include +#include + +#include "util/rstime.h" + +#ifndef XAPIAN_AT_LEAST +#define XAPIAN_AT_LEAST(A,B,C) (XAPIAN_MAJOR_VERSION > (A) || \ + (XAPIAN_MAJOR_VERSION == (A) && \ + (XAPIAN_MINOR_VERSION > (B) || \ + (XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C))))) +#endif // ndef XAPIAN_AT_LEAST + +namespace DeepSearch +{ + +std::unique_ptr openWritableDatabase( + const std::string& path, int flags = 0, int blockSize = 0 ); + +std::unique_ptr openReadOnlyDatabase( + const std::string& path, int flags = 0 ); + +std::string timetToXapianDate(const rstime_t& time); + +} diff --git a/libretroshare/src/deep_search/deep_search.h b/libretroshare/src/deep_search/deep_search.h deleted file mode 100644 index b67c93055..000000000 --- a/libretroshare/src/deep_search/deep_search.h +++ /dev/null @@ -1,276 +0,0 @@ -/******************************************************************************* - * libretroshare/src/crypto: crypto.h * - * * - * libretroshare: retroshare core library * - * * - * Copyright (C) 2018 Gioacchino Mazzurco * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public License * - * along with this program. If not, see . * - * * - *******************************************************************************/ -#pragma once - -#include "util/rstime.h" -#include -#include - -#include "retroshare/rsgxschannels.h" -#include "retroshare/rsinit.h" -#include "util/rsurl.h" - -#ifndef XAPIAN_AT_LEAST -#define XAPIAN_AT_LEAST(A,B,C) (XAPIAN_MAJOR_VERSION > (A) || \ - (XAPIAN_MAJOR_VERSION == (A) && \ - (XAPIAN_MINOR_VERSION > (B) || \ - (XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C))))) -#endif // ndef XAPIAN_AT_LEAST - -struct DeepSearch -{ - struct SearchResult - { - std::string mUrl; - std::string mSnippet; - }; - - /** - * @param[in] maxResults maximum number of acceptable search results, 0 for - * no limits - * @return search results count - */ - static uint32_t search( const std::string& queryStr, - std::vector& results, - uint32_t maxResults = 100 ) - { - results.clear(); - - Xapian::Database db; - - // Open the database we're going to search. - try { db = Xapian::Database(dbPath()); } - catch(Xapian::DatabaseOpeningError e) - { - std::cerr << __PRETTY_FUNCTION__ << " " << e.get_msg() - << ", probably nothing has been indexed yet."<< std::endl; - return 0; - } - catch(Xapian::DatabaseError e) - { - std::cerr << __PRETTY_FUNCTION__ << " " << e.get_msg() - << " this is fishy, maybe " << dbPath() - << " has been corrupted (deleting it may help in that " - << "case without loosing data)" << std::endl; - return 0; - } - - // Set up a QueryParser with a stemmer and suitable prefixes. - Xapian::QueryParser queryparser; - //queryparser.set_stemmer(Xapian::Stem("en")); - queryparser.set_stemming_strategy(queryparser.STEM_SOME); - // Start of prefix configuration. - //queryparser.add_prefix("title", "S"); - //queryparser.add_prefix("description", "XD"); - // End of prefix configuration. - - // And parse the query. - Xapian::Query query = queryparser.parse_query(queryStr); - - // Use an Enquire object on the database to run the query. - Xapian::Enquire enquire(db); - enquire.set_query(query); - - Xapian::MSet mset = enquire.get_mset( - 0, maxResults ? maxResults : db.get_doccount() ); - - for ( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m ) - { - const Xapian::Document& doc = m.get_document(); - SearchResult s; - s.mUrl = doc.get_value(URL_VALUENO); -#if XAPIAN_AT_LEAST(1,3,5) - s.mSnippet = mset.snippet(doc.get_data()); -#endif // XAPIAN_AT_LEAST(1,3,5) - results.push_back(s); - } - - return results.size(); - } - - - static void indexChannelGroup(const RsGxsChannelGroup& chan) - { - Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN); - - // Set up a TermGenerator that we'll use in indexing. - Xapian::TermGenerator termgenerator; - //termgenerator.set_stemmer(Xapian::Stem("en")); - - // We make a document and tell the term generator to use this. - Xapian::Document doc; - termgenerator.set_document(doc); - - // Index each field with a suitable prefix. - termgenerator.index_text(chan.mMeta.mGroupName, 1, "G"); - termgenerator.index_text(timetToXapianDate(chan.mMeta.mPublishTs), 1, "D"); - termgenerator.index_text(chan.mDescription, 1, "XD"); - - // Index fields without prefixes for general search. - termgenerator.index_text(chan.mMeta.mGroupName); - termgenerator.increase_termpos(); - termgenerator.index_text(chan.mDescription); - - RsUrl chanUrl; chanUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", chan.mMeta.mGroupId.toStdString()); - const std::string idTerm("Q" + chanUrl.toString()); - - chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs)); - chanUrl.setQueryKV("name", chan.mMeta.mGroupName); - if(!chan.mMeta.mAuthorId.isNull()) - chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString()); - if(chan.mMeta.mSignFlags) - chanUrl.setQueryKV( "signFlags", - std::to_string(chan.mMeta.mSignFlags) ); - std::string rsLink(chanUrl.toString()); - - // store the RS link so we are able to retrive it on matching search - doc.add_value(URL_VALUENO, rsLink); - - // Store some fields for display purposes. - doc.set_data(chan.mMeta.mGroupName + "\n" + chan.mDescription); - - // We use the identifier to ensure each object ends up in the - // database only once no matter how many times we run the - // indexer. "Q" prefix is a Xapian convention for unique id term. - doc.add_boolean_term(idTerm); - db.replace_document(idTerm, doc); - } - - static void removeChannelFromIndex(RsGxsGroupId grpId) - { - // "Q" prefix is a Xapian convention for unique id term. - RsUrl chanUrl; chanUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", grpId.toStdString()); - std::string idTerm("Q" + chanUrl.toString()); - - Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN); - db.delete_document(idTerm); - } - - static void indexChannelPost(const RsGxsChannelPost& post) - { - Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN); - - // Set up a TermGenerator that we'll use in indexing. - Xapian::TermGenerator termgenerator; - //termgenerator.set_stemmer(Xapian::Stem("en")); - - // We make a document and tell the term generator to use this. - Xapian::Document doc; - termgenerator.set_document(doc); - - // Index each field with a suitable prefix. - termgenerator.index_text(post.mMeta.mMsgName, 1, "S"); - termgenerator.index_text(timetToXapianDate(post.mMeta.mPublishTs), 1, "D"); - - // Avoid indexing HTML - bool isPlainMsg = post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>'; - - if(isPlainMsg) - termgenerator.index_text(post.mMsg, 1, "XD"); - - // Index fields without prefixes for general search. - termgenerator.index_text(post.mMeta.mMsgName); - if(isPlainMsg) - { - termgenerator.increase_termpos(); - termgenerator.index_text(post.mMsg); - } - - for(const RsGxsFile& attachment : post.mFiles) - { - termgenerator.index_text(attachment.mName, 1, "F"); - - termgenerator.increase_termpos(); - termgenerator.index_text(attachment.mName); - } - - // We use the identifier to ensure each object ends up in the - // database only once no matter how many times we run the - // indexer. - RsUrl postUrl; postUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", post.mMeta.mGroupId.toStdString()) - .setQueryKV("msgid", post.mMeta.mMsgId.toStdString()); - std::string idTerm("Q" + postUrl.toString()); - - postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs)); - postUrl.setQueryKV("name", post.mMeta.mMsgName); - postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString()); - std::string rsLink(postUrl.toString()); - - // store the RS link so we are able to retrive it on matching search - doc.add_value(URL_VALUENO, rsLink); - - // Store some fields for display purposes. - if(isPlainMsg) - doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg); - else doc.set_data(post.mMeta.mMsgName); - - doc.add_boolean_term(idTerm); - db.replace_document(idTerm, doc); - } - - static void removeChannelPostFromIndex( - RsGxsGroupId grpId, RsGxsMessageId msgId ) - { - RsUrl postUrl; postUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", grpId.toStdString()) - .setQueryKV("msgid", msgId.toStdString()); - // "Q" prefix is a Xapian convention for unique id term. - std::string idTerm("Q" + postUrl.toString()); - - Xapian::WritableDatabase db(dbPath(), Xapian::DB_CREATE_OR_OPEN); - db.delete_document(idTerm); - } - -private: - - enum : Xapian::valueno - { - /// Used to store retroshare url of indexed documents - URL_VALUENO, - - /// @see Xapian::BAD_VALUENO - BAD_VALUENO = Xapian::BAD_VALUENO - }; - - static const std::string& dbPath() - { - static const std::string dbDir = - RsAccounts::AccountDirectory() + "/deep_search_xapian_db"; - return dbDir; - } - - static std::string timetToXapianDate(const rstime_t& time) - { - char date[] = "YYYYMMDD\0"; - time_t tTime = static_cast(time); - std::strftime(date, 9, "%Y%m%d", std::gmtime(&tTime)); - return date; - } -}; - diff --git a/libretroshare/src/deep_search/filesflacindexer.hpp b/libretroshare/src/deep_search/filesflacindexer.hpp new file mode 100644 index 000000000..dbdc62b99 --- /dev/null +++ b/libretroshare/src/deep_search/filesflacindexer.hpp @@ -0,0 +1,156 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "deep_search/filesindex.hpp" +#include "util/rsdebug.h" + +#include +#include +#include +#include +#include + +struct RsDeepFlacFileIndexer +{ + RsDeepFlacFileIndexer() + { + DeepFilesIndex::registerIndexer(31, indexFlacFile); + } + + static uint32_t indexFlacFile( + const std::string& path, const std::string& /*name*/, + Xapian::TermGenerator& xTG, Xapian::Document& xDoc ) + { + Dbg3() << __PRETTY_FUNCTION__ << " " << path << std::endl; + + using FlacChain = FLAC::Metadata::Chain; + std::unique_ptr flacChain(new FlacChain); + + if(!flacChain->is_valid()) + { + RsErr() << __PRETTY_FUNCTION__ << " Failed creating FLAC Chain 1" + << std::endl; + return 1; + } + + if(!flacChain->read(path.c_str(), false)) + { + Dbg3() << __PRETTY_FUNCTION__ << " Failed to open the file as FLAC" + << std::endl; + + flacChain.reset(new FlacChain); + if(!flacChain->is_valid()) + { + RsErr() << __PRETTY_FUNCTION__ + << " Failed creating FLAC Chain 2" << std::endl; + return 1; + } + if(!flacChain->read(path.c_str(), true)) + { + Dbg3() << __PRETTY_FUNCTION__ + << " Failed to open the file as OggFLAC" + << std::endl; + return 0; + } + } + + unsigned validCommentsCnt = 0; + std::string docData = xDoc.get_data(); + + FLAC::Metadata::Iterator mdit; + mdit.init(*flacChain); + if(!mdit.is_valid()) return 1; + + do + { + ::FLAC__MetadataType mdt = mdit.get_block_type(); + if (mdt != FLAC__METADATA_TYPE_VORBIS_COMMENT) continue; + + Dbg2() << __PRETTY_FUNCTION__ << " Found Vorbis Comment Block" + << std::endl; + + std::unique_ptr proto(mdit.get_block()); + if(!proto) continue; + + const FLAC::Metadata::VorbisComment* vc = + dynamic_cast(proto.get()); + if(!vc || !vc->is_valid()) continue; + + unsigned numComments = vc->get_num_comments(); + for(unsigned i = 0; i < numComments; ++i) + { + FLAC::Metadata::VorbisComment::Entry entry = + vc->get_comment(i); + if(!entry.is_valid()) continue; + + std::string tagName( entry.get_field_name(), + entry.get_field_name_length() ); + + /* Vorbis tags should be uppercases but not all the softwares + * enforce it */ + for (auto& c: tagName) c = static_cast(toupper(c)); + + std::string tagValue( entry.get_field_value(), + entry.get_field_value_length() ); + + if(tagValue.empty()) continue; + + if(tagName == "ARTIST") + xTG.index_text(tagValue, 1, "A"); + else if (tagName == "DESCRIPTION") + xTG.index_text(tagValue, 1, "XD"); + else if (tagName == "TITLE") + xTG.index_text(tagValue, 1, "S"); + else if(tagName.find("COVERART") != tagName.npos) + continue; // Avoid polluting the index with binary data + else if (tagName.find("METADATA_BLOCK_PICTURE") != tagName.npos) + continue; // Avoid polluting the index with binary data + + // Index fields without prefixes for general search. + xTG.increase_termpos(); + std::string fullComment(tagName + "=" + tagValue); + xTG.index_text(fullComment); + docData += fullComment + "\n"; + + Dbg2() << __PRETTY_FUNCTION__ << " Indexed " << fullComment + << std::endl; + + ++validCommentsCnt; + } + } + while(mdit.next()); + + if(validCommentsCnt > 0) + { + Dbg1() << __PRETTY_FUNCTION__ << " Successfully indexed: " << path + << std::endl; + + xDoc.set_data(docData); + return 99; + } + + /* Altought the file appears to be a valid FLAC, no vorbis comment has + * been found so return less then 50 maybe it has tagged only with ID3 + * tags ? */ + return 30; + } + + RS_SET_CONTEXT_DEBUG_LEVEL(3) +}; diff --git a/libretroshare/src/deep_search/filesindex.cpp b/libretroshare/src/deep_search/filesindex.cpp new file mode 100644 index 000000000..3edcf9a97 --- /dev/null +++ b/libretroshare/src/deep_search/filesindex.cpp @@ -0,0 +1,171 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "deep_search/filesindex.hpp" +#include "deep_search/commonutils.hpp" +#include "util/rsdebug.h" +#include "retroshare/rsinit.h" +#include "retroshare/rsversion.h" + +#include + +/*static*/ std::multimap +DeepFilesIndex::indexersRegister = {}; + +bool DeepFilesIndex::indexFile( + const std::string& path, const std::string& name, + const RsFileHash& hash ) +{ + auto dbPtr = DeepSearch::openWritableDatabase( + mDbPath, Xapian::DB_CREATE_OR_OPEN ); + if(!dbPtr) return false; + Xapian::WritableDatabase& db(*dbPtr); + + const std::string hashString = hash.toStdString(); + const std::string idTerm("Q" + hashString); + + Xapian::Document oldDoc; + Xapian::PostingIterator pIt = db.postlist_begin(idTerm); + if( pIt != db.postlist_end(idTerm) ) + { + oldDoc = db.get_document(*pIt); + if( oldDoc.get_value(INDEXER_VERSION_VALUENO) == + RS_HUMAN_READABLE_VERSION && + std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) == + indexersRegister.size() ) + { + /* Looks like this file has already been indexed by this RetroShare + * exact version, so we can skip it. If the version was different it + * made sense to reindex it as better indexers might be available + * since last time it was indexed */ + Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: " + << hash << " " << name << std::endl; + return true; + } + } + + Xapian::Document doc; + + // Set up a TermGenerator that we'll use in indexing. + Xapian::TermGenerator termgenerator; + //termgenerator.set_stemmer(Xapian::Stem("en")); + termgenerator.set_document(doc); + + for(auto& indexerPair : indexersRegister) + if(indexerPair.second(path, name, termgenerator, doc) > 50) + break; + + doc.add_boolean_term(idTerm); + termgenerator.index_text(name, 1, "N"); + termgenerator.index_text(name); + doc.add_value(FILE_HASH_VALUENO, hashString); + doc.add_value(INDEXER_VERSION_VALUENO, RS_HUMAN_READABLE_VERSION); + doc.add_value( + INDEXERS_COUNT_VALUENO, + std::to_string(indexersRegister.size()) ); + db.replace_document(idTerm, doc); + + return true; +} + +bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash) +{ + Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: " + << hash << std::endl; + + std::unique_ptr db = + DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN); + if(!db) return false; + + db->delete_document("Q" + hash.toStdString()); + return true; +} + +/*static*/ std::string DeepFilesIndex::dbDefaultPath() +{ return RsAccounts::AccountDirectory() + "/deep_files_index_xapian_db"; } + +/*static*/ bool DeepFilesIndex::registerIndexer( + int order, const DeepFilesIndex::IndexerFunType& indexerFun ) +{ + Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl; + + indexersRegister.insert(std::make_pair(order, indexerFun)); + return true; +} + +uint32_t DeepFilesIndex::search( + const std::string& queryStr, + std::vector& results, uint32_t maxResults ) +{ + results.clear(); + + auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath); + if(!dbPtr) return 0; + Xapian::Database& db(*dbPtr); + + // Set up a QueryParser with a stemmer and suitable prefixes. + Xapian::QueryParser queryparser; + //queryparser.set_stemmer(Xapian::Stem("en")); + queryparser.set_stemming_strategy(queryparser.STEM_SOME); + // Start of prefix configuration. + //queryparser.add_prefix("title", "S"); + //queryparser.add_prefix("description", "XD"); + // End of prefix configuration. + + // And parse the query. + Xapian::Query query = queryparser.parse_query(queryStr); + + // Use an Enquire object on the database to run the query. + Xapian::Enquire enquire(db); + enquire.set_query(query); + + Xapian::MSet mset = enquire.get_mset( + 0, maxResults ? maxResults : db.get_doccount() ); + + for ( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m ) + { + const Xapian::Document& doc = m.get_document(); + DeepFilesSearchResult s; + s.mFileHash = RsFileHash(doc.get_value(FILE_HASH_VALUENO)); + s.mWeight = m.get_weight(); +#if XAPIAN_AT_LEAST(1,3,5) + s.mSnippet = mset.snippet(doc.get_data()); +#endif // XAPIAN_AT_LEAST(1,3,5) + results.push_back(s); + } + + return static_cast(results.size()); +} + + +#ifdef RS_DEEP_FILES_INDEX_OGG +# include "deep_search/filesoggindexer.hpp" +static RsDeepOggFileIndexer oggFileIndexer; +#endif // def RS_DEEP_FILES_INDEX_OGG + +#ifdef RS_DEEP_FILES_INDEX_FLAC +# include "deep_search/filesflacindexer.hpp" +static RsDeepFlacFileIndexer flacFileIndexer; +#endif // def RS_DEEP_FILES_INDEX_FLAC + +#ifdef RS_DEEP_FILES_INDEX_TAGLIB +# include "deep_search/filestaglibindexer.hpp" +static RsDeepTaglibFileIndexer taglibFileIndexer; +#endif // def RS_DEEP_FILES_INDEX_TAGLIB diff --git a/libretroshare/src/deep_search/filesindex.hpp b/libretroshare/src/deep_search/filesindex.hpp new file mode 100644 index 000000000..f811e5e9c --- /dev/null +++ b/libretroshare/src/deep_search/filesindex.hpp @@ -0,0 +1,103 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +#include "retroshare/rstypes.h" +#include "util/rsdebug.h" + +#include +#include +#include +#include +#include +#include + +struct DeepFilesSearchResult +{ + DeepFilesSearchResult() : mWeight(0) {} + + RsFileHash mFileHash; + double mWeight; + std::string mSnippet; +}; + +class DeepFilesIndex +{ +public: + DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {} + + /** + * @brief Search indexed files + * @param[in] maxResults maximum number of acceptable search results, 0 for + * no limits + * @return search results count + */ + uint32_t search( const std::string& queryStr, + std::vector& results, + uint32_t maxResults = 100 ); + + /** + * @return false if file could not be indexed because of error or + * unsupported type, true otherwise. + */ + bool indexFile( + const std::string& path, const std::string& name, + const RsFileHash& hash ); + + /** + * @brief Remove file entry from database + * @return false on error, true otherwise. + */ + bool removeFileFromIndex(const RsFileHash& hash); + + static std::string dbDefaultPath(); + + using IndexerFunType = std::function< + uint32_t( const std::string& path, const std::string& name, + Xapian::TermGenerator& xTG, Xapian::Document& xDoc ) >; + + static bool registerIndexer( + int order, const IndexerFunType& indexerFun ); + +private: + enum : Xapian::valueno + { + /// Used to store RsFileHash of indexed documents + FILE_HASH_VALUENO, + + /** Used to check if some file need reindex because was indexed with an + * older version of the indexer */ + INDEXER_VERSION_VALUENO, + + /** Used to check if some file need reindex because was indexed with an + * older version of the indexer */ + INDEXERS_COUNT_VALUENO, + + /// @see Xapian::BAD_VALUENO + BAD_VALUENO = Xapian::BAD_VALUENO + }; + + const std::string mDbPath; + + /** Storage for indexers function by order */ + static std::multimap indexersRegister; + + RS_SET_CONTEXT_DEBUG_LEVEL(1) +}; diff --git a/libretroshare/src/deep_search/filesoggindexer.hpp b/libretroshare/src/deep_search/filesoggindexer.hpp new file mode 100644 index 000000000..00c22ae66 --- /dev/null +++ b/libretroshare/src/deep_search/filesoggindexer.hpp @@ -0,0 +1,97 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "deep_search/filesindex.hpp" +#include "util/rsdebug.h" + +#include +#include +#include +#include + +struct RsDeepOggFileIndexer +{ + RsDeepOggFileIndexer() + { + DeepFilesIndex::registerIndexer(30, indexOggFile); + } + + static uint32_t indexOggFile( + const std::string& path, const std::string& /*name*/, + Xapian::TermGenerator& xTG, Xapian::Document& xDoc ) + { + Dbg3() << __PRETTY_FUNCTION__ << " " << path << std::endl; + + OggVorbis_File vf; + int ret = ov_fopen(path.c_str(), &vf); + + if(ret == 0 && vf.vc) + { + vorbis_comment& vc = *vf.vc; + std::string docData = xDoc.get_data(); + for (int i = 0; i < vc.comments; ++i) + { + using szt = std::string::size_type; + std::string userComment( + vc.user_comments[i], + static_cast(vc.comment_lengths[i]) ); + + if(userComment.empty()) continue; + + szt equalPos = userComment.find('='); + if(equalPos == std::string::npos) continue; + + std::string tagName = userComment.substr(0, equalPos); + if(tagName.empty()) continue; + + std::string tagValue = userComment.substr(equalPos + 1); + if(tagValue.empty()) continue; + + /* Ogg tags should be uppercases but not all the softwares + * enforce it */ + for (auto& c: tagName) c = static_cast(toupper(c)); + + if(tagName == "ARTIST") + xTG.index_text(tagValue, 1, "A"); + else if (tagName == "DESCRIPTION") + xTG.index_text(tagValue, 1, "XD"); + else if (tagName == "TITLE") + xTG.index_text(tagValue, 1, "S"); + else if(tagName.find("COVERART") != tagName.npos) + continue; // Avoid polluting the index with binary data + else if (tagName.find("METADATA_BLOCK_PICTURE") != tagName.npos) + continue; // Avoid polluting the index with binary data + + // Index fields without prefixes for general search. + xTG.increase_termpos(); + xTG.index_text(userComment); + docData += userComment + "\n"; + } + xDoc.set_data(docData); + + ov_clear(&vf); + return 99; + } + + return 0; + } + + RS_SET_CONTEXT_DEBUG_LEVEL(1) +}; diff --git a/libretroshare/src/deep_search/filestaglibindexer.hpp b/libretroshare/src/deep_search/filestaglibindexer.hpp new file mode 100644 index 000000000..341e9af38 --- /dev/null +++ b/libretroshare/src/deep_search/filestaglibindexer.hpp @@ -0,0 +1,103 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "deep_search/filesindex.hpp" +#include "util/rsdebug.h" + +#include +#include +#include +#include +#include +#include + +struct RsDeepTaglibFileIndexer +{ + RsDeepTaglibFileIndexer() + { + DeepFilesIndex::registerIndexer(40, indexFile); + } + + static uint32_t indexFile( + const std::string& path, const std::string& /*name*/, + Xapian::TermGenerator& xTG, Xapian::Document& xDoc ) + { + Dbg4() << __PRETTY_FUNCTION__ << " " << path << std::endl; + + TagLib::FileRef tFile(path.c_str()); + if(tFile.isNull()) return 0; + + const TagLib::Tag* tag = tFile.tag(); + if(!tag) return 0; + + TagLib::PropertyMap tMap = tag->properties(); + + unsigned validCommentsCnt = 0; + std::string docData = xDoc.get_data(); + for( TagLib::PropertyMap::ConstIterator mIt = tMap.begin(); + mIt != tMap.end(); ++mIt ) + { + if(mIt->first.isNull() || mIt->first.isEmpty()) continue; + std::string tagName(mIt->first.upper().to8Bit()); + + if(mIt->second.isEmpty()) continue; + std::string tagValue(mIt->second.toString(", ").to8Bit(true)); + if(tagValue.empty()) continue; + + if(tagName == "ARTIST") + xTG.index_text(tagValue, 1, "A"); + else if (tagName == "DESCRIPTION") + xTG.index_text(tagValue, 1, "XD"); + else if (tagName == "TITLE") + xTG.index_text(tagValue, 1, "S"); + else if(tagName.find("COVERART") != tagName.npos) + continue; // Avoid polluting the index with binary data + else if (tagName.find("METADATA_BLOCK_PICTURE") != tagName.npos) + continue; // Avoid polluting the index with binary data + + // Index fields without prefixes for general search. + xTG.increase_termpos(); + std::string fullComment(tagName + "=" + tagValue); + xTG.index_text(fullComment); + docData += fullComment + "\n"; + + Dbg2() << __PRETTY_FUNCTION__ << " Indexed " << tagName << "=\"" + << tagValue << '"' << std::endl; + + ++validCommentsCnt; + } + + if(validCommentsCnt > 0) + { + Dbg1() << __PRETTY_FUNCTION__ << " Successfully indexed: " << path + << std::endl; + + xDoc.set_data(docData); + return 99; + } + + /* Altought the file appears to be supported by taglib, no comments has + * been found so return less then 50 maybe another indexer is capable of + * extracting information */ + return 30; + } + + RS_SET_CONTEXT_DEBUG_LEVEL(3) +}; diff --git a/libretroshare/src/file_sharing/dir_hierarchy.cc b/libretroshare/src/file_sharing/dir_hierarchy.cc index 6522da2cd..7d7864165 100644 --- a/libretroshare/src/file_sharing/dir_hierarchy.cc +++ b/libretroshare/src/file_sharing/dir_hierarchy.cc @@ -21,15 +21,19 @@ ******************************************************************************/ #include #include + #include "util/rstime.h" #include "util/rsdir.h" #include "util/rsprint.h" #include "retroshare/rsexpr.h" - #include "dir_hierarchy.h" #include "filelist_io.h" #include "file_sharing_defaults.h" +#ifdef RS_DEEP_FILES_INDEX +# include "deep_search/filesindex.hpp" +#endif // def RS_DEEP_FILES_INDEX + //#define DEBUG_DIRECTORY_STORAGE 1 typedef FileListIO::read_error read_error; @@ -391,6 +395,11 @@ void InternalFileHierarchyStorage::deleteFileNode(uint32_t index) { FileEntry& fe(*static_cast(mNodes[index])) ; +#ifdef RS_DEEP_FILES_INDEX + DeepFilesIndex tfi(DeepFilesIndex::dbDefaultPath()); + tfi.removeFileFromIndex(fe.file_hash); +#endif + if(mTotalSize >= fe.file_size) mTotalSize -= fe.file_size ; @@ -753,7 +762,9 @@ int InternalFileHierarchyStorage::searchBoolExp(RsRegularExpression::Expression return 0; } -int InternalFileHierarchyStorage::searchTerms(const std::list& terms, std::list &results) const +int InternalFileHierarchyStorage::searchTerms( + const std::list& terms, + std::list& results ) const { // most entries are likely to be files, so we could do a linear search over the entries tab. // instead we go through the table of hashes. diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 2e328e61e..fbbb27e4a 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -20,6 +20,7 @@ * * ******************************************************************************/ #include + #include "util/rstime.h" #include "serialiser/rstlvbinary.h" #include "retroshare/rspeers.h" @@ -30,6 +31,10 @@ #include "dir_hierarchy.h" #include "filelist_io.h" +#ifdef RS_DEEP_FILES_INDEX +# include "deep_search/filesindex.hpp" +#endif // def RS_DEEP_FILES_INDEX + //#define DEBUG_REMOTE_DIRECTORY_STORAGE 1 /******************************************************************************************************************/ @@ -180,7 +185,9 @@ void DirectoryStorage::print() mFileHierarchy->print(); } -int DirectoryStorage::searchTerms(const std::list& terms, std::list &results) const +int DirectoryStorage::searchTerms( + const std::list& terms, + std::list& results ) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->searchTerms(terms,results); @@ -501,18 +508,39 @@ void LocalDirectoryStorage::updateTimeStamps() #endif } } -bool LocalDirectoryStorage::updateHash(const EntryIndex& index, const RsFileHash& hash, bool update_internal_hierarchy) -{ - RS_STACK_MUTEX(mDirStorageMtx) ; - mEncryptedHashes[makeEncryptedHash(hash)] = hash ; - mChanged = true ; +bool LocalDirectoryStorage::updateHash( + const EntryIndex& index, const RsFileHash& hash, + bool update_internal_hierarchy ) +{ + bool ret = false; + + { + RS_STACK_MUTEX(mDirStorageMtx); + + mEncryptedHashes[makeEncryptedHash(hash)] = hash ; + mChanged = true ; #ifdef DEBUG_LOCAL_DIRECTORY_STORAGE - std::cerr << "Updating index of hash " << hash << " update_internal=" << update_internal_hierarchy << std::endl; + std::cerr << "Updating index of hash " << hash << " update_internal=" + << update_internal_hierarchy << std::endl; #endif - return (!update_internal_hierarchy)|| mFileHierarchy->updateHash(index,hash); + ret = (!update_internal_hierarchy) || + mFileHierarchy->updateHash(index,hash); + } // RS_STACK_MUTEX(mDirStorageMtx); + +#ifdef RS_DEEP_FILES_INDEX + FileInfo fInfo; + if( ret && getFileInfo(index, fInfo) && + fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH ) + { + DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath()); + ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash); + } +#endif // def RS_DEEP_FILES_INDEX + + return ret; } std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const { diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 132973bfd..d52340de0 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -54,6 +54,10 @@ #include #include "util/rstime.h" +#ifdef RS_DEEP_FILES_INDEX +# include "deep_search/filesindex.hpp" +#endif // def RS_DEEP_FILES_INDEX + /*** * #define SERVER_DEBUG 1 * #define SERVER_DEBUG_CACHE 1 @@ -65,9 +69,26 @@ static const rstime_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priority tasks handling every 5 seconds static const rstime_t FILE_TRANSFER_MAX_DELAY_BEFORE_DROP_USAGE_RECORD = 10 ; // keep usage records for 10 secs at most. +#ifdef RS_DEEP_FILES_INDEX +TurtleFileInfoV2::TurtleFileInfoV2(const DeepFilesSearchResult& dRes) : + fHash(dRes.mFileHash), fWeight(static_cast(dRes.mWeight)), + fSnippet(dRes.mSnippet) +{ + FileInfo fInfo; + rsFiles->FileDetails(fHash, RS_FILE_HINTS_LOCAL, fInfo); + + fSize = fInfo.size; + fName = fInfo.fname; +} +#endif // def RS_DEEP_FILES_INDEX + +TurtleFileInfoV2::~TurtleFileInfoV2() = default; + /* Setup */ -ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc) - : p3Service(),RsServiceSerializer(RS_SERVICE_TYPE_TURTLE), // should be FT, but this is for backward compatibility +ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc): + p3Service(), + // should be FT, but this is for backward compatibility + RsServiceSerializer(RS_SERVICE_TYPE_TURTLE), mPeerMgr(pm), mServiceCtrl(sc), mFileDatabase(NULL), mFtController(NULL), mFtExtra(NULL), @@ -500,15 +521,24 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi return false; } -RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const +RsItem *ftServer::create_item(uint16_t service, uint8_t item_type) const { #ifdef SERVER_DEBUG FTSERVER_DEBUG() << "p3turtle: deserialising packet: " << std::endl ; #endif - if (RS_SERVICE_TYPE_TURTLE != service) + + RsServiceType serviceType = static_cast(service); + switch (serviceType) { - FTSERVER_ERROR() << " Wrong type !!" << std::endl ; - return NULL; /* wrong type */ + /* This one is here for retro-compatibility as turtle routing and file + * trasfer services were just one service before turle service was + * generalized */ + case RsServiceType::TURTLE: break; + case RsServiceType::FILE_TRANSFER: break; + default: + RsErr() << __PRETTY_FUNCTION__ << " Wrong service type: " << service + << std::endl; + return nullptr; } try @@ -521,16 +551,19 @@ RsItem *ftServer::create_item(uint16_t service,uint8_t item_type) const case RS_TURTLE_SUBTYPE_FILE_MAP : return new RsTurtleFileMapItem(); case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST : return new RsTurtleChunkCrcRequestItem(); case RS_TURTLE_SUBTYPE_CHUNK_CRC : return new RsTurtleChunkCrcItem(); - + case static_cast(RsFileItemType::FILE_SEARCH_REQUEST): + return new RsFileSearchRequestItem(); + case static_cast(RsFileItemType::FILE_SEARCH_RESULT): + return new RsFileSearchResultItem(); default: - return NULL ; + return nullptr; } } catch(std::exception& e) { FTSERVER_ERROR() << "(EE) deserialisation error in " << __PRETTY_FUNCTION__ << ": " << e.what() << std::endl; - return NULL ; + return nullptr; } } @@ -1837,7 +1870,12 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item) if(cbpt != mSearchCallbacksMap.end()) { hasCallback = true; - cbpt->second.first(item->result); + + std::vector cRes; + for( const auto& tfiold : item->result) + cRes.push_back(tfiold); + + cbpt->second.first(cRes); } } // end RS_STACK_MUTEX(mSearchCallbacksMapMutex); @@ -1845,6 +1883,99 @@ void ftServer::ftReceiveSearchResult(RsTurtleFTSearchResultItem *item) RsServer::notify()->notifyTurtleSearchResult(item->PeerId(),item->request_id, item->result ); } +bool ftServer::receiveSearchRequest( + unsigned char* searchRequestData, uint32_t searchRequestDataLen, + unsigned char*& searchResultData, uint32_t& searchResultDataLen, + uint32_t& maxAllowsHits ) +{ +#ifdef RS_DEEP_FILES_INDEX + std::unique_ptr recvItem( + RsServiceSerializer::deserialise( + searchRequestData, &searchRequestDataLen ) ); + + if(!recvItem) + { + RsWarn() << __PRETTY_FUNCTION__ << " Search request deserialization " + << "failed" << std::endl; + return false; + } + + std::unique_ptr sReqItPtr( + dynamic_cast(recvItem.get()) ); + if(!sReqItPtr) + { + RsWarn() << __PRETTY_FUNCTION__ << " Received an invalid search request" + << " " << *recvItem << std::endl; + return false; + } + recvItem.release(); + + RsFileSearchRequestItem& searchReq(*sReqItPtr); + + std::vector dRes; + DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath()); + if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0) + { + RsFileSearchResultItem resIt; + + for(const auto& dMatch : dRes) + resIt.mResults.push_back(TurtleFileInfoV2(dMatch)); + + searchResultDataLen = RsServiceSerializer::size(&resIt); + searchResultData = static_cast(malloc(searchResultDataLen)); + return RsServiceSerializer::serialise( + &resIt, searchResultData, &searchResultDataLen ); + } +#endif // def RS_DEEP_FILES_INDEX + + searchResultData = nullptr; + searchResultDataLen = 0; + return false; +} + +void ftServer::receiveSearchResult( + TurtleSearchRequestId requestId, unsigned char* searchResultData, + uint32_t searchResultDataLen ) +{ + if(!searchResultData || !searchResultDataLen) + { + RsWarn() << __PRETTY_FUNCTION__ << " got null paramethers " + << "searchResultData: " << static_cast(searchResultData) + << " searchResultDataLen: " << searchResultDataLen + << " seems someone else in the network have a buggy RetroShare" + << " implementation" << std::endl; + return; + } + + RS_STACK_MUTEX(mSearchCallbacksMapMutex); + auto cbpt = mSearchCallbacksMap.find(requestId); + if(cbpt != mSearchCallbacksMap.end()) + { + RsItem* recvItem = RsServiceSerializer::deserialise( + searchResultData, &searchResultDataLen ); + + if(!recvItem) + { + RsWarn() << __PRETTY_FUNCTION__ << " Search result deserialization " + << "failed" << std::endl; + return; + } + + std::unique_ptr resItPtr( + dynamic_cast(recvItem) ); + + if(!resItPtr) + { + RsWarn() << __PRETTY_FUNCTION__ << " Received invalid search result" + << std::endl; + delete recvItem; + return; + } + + cbpt->second.first(resItPtr->mResults); + } +} + /***************************** CONFIG ****************************/ bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr) @@ -1857,27 +1988,74 @@ bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr) return true; } +#ifdef RS_DEEP_FILES_INDEX +static std::vector xapianQueryKeywords = +{ + " AND ", " OR ", " NOT ", " XOR ", " +", " -", " ( ", " ) ", " NEAR ", + " ADJ ", " \"", "\" " +}; +#endif + bool ftServer::turtleSearchRequest( const std::string& matchString, - const std::function& results)>& multiCallback, + const std::function& results)>& multiCallback, rstime_t maxWait ) { if(matchString.empty()) { - std::cerr << __PRETTY_FUNCTION__ << " match string can't be empty!" - << std::endl; + RsWarn() << __PRETTY_FUNCTION__ << " match string can't be empty!" + << std::endl; return false; } - TurtleRequestId sId = turtleSearch(matchString); +#ifdef RS_DEEP_FILES_INDEX + RsFileSearchRequestItem sItem; + sItem.queryString = matchString; - RS_STACK_MUTEX(mSearchCallbacksMapMutex); - mSearchCallbacksMap.emplace( - sId, - std::make_pair( - multiCallback, - std::chrono::system_clock::now() + - std::chrono::seconds(maxWait) ) ); + uint32_t iSize = RsServiceSerializer::size(&sItem); + uint8_t* iBuf = static_cast(malloc(iSize)); + RsServiceSerializer::serialise(&sItem, iBuf, &iSize); + + Dbg3() << __PRETTY_FUNCTION__ << " sending search request:" << sItem + << std::endl; + + TurtleRequestId xsId = mTurtleRouter->turtleSearch(iBuf, iSize, this); + + { RS_STACK_MUTEX(mSearchCallbacksMapMutex); + mSearchCallbacksMap.emplace( + xsId, + std::make_pair( + multiCallback, + std::chrono::system_clock::now() + + std::chrono::seconds(maxWait) ) ); + } // RS_STACK_MUTEX(mSearchCallbacksMapMutex); + + /* Trick to keep receiving more or less usable results from old peers */ + std::string strippedQuery = matchString; + for(const std::string& xKeyword : xapianQueryKeywords) + { + std::string::size_type pos = std::string::npos; + while( (pos = strippedQuery.find(xKeyword)) != std::string::npos ) + strippedQuery.replace(pos, xKeyword.length(), " "); + } + + Dbg3() << __PRETTY_FUNCTION__ << " sending stripped query for " + << "retro-compatibility: " << strippedQuery << std::endl; + + TurtleRequestId sId = mTurtleRouter->turtleSearch(strippedQuery); +#else // def RS_DEEP_FILES_INDEX + TurtleRequestId sId = mTurtleRouter->turtleSearch(matchString); +#endif // def RS_DEEP_FILES_INDEX + + { + RS_STACK_MUTEX(mSearchCallbacksMapMutex); + mSearchCallbacksMap.emplace( + sId, + std::make_pair( + multiCallback, + std::chrono::system_clock::now() + + std::chrono::seconds(maxWait) ) ); + } return true; } @@ -1913,4 +2091,13 @@ bool ftServer::isHashBanned(const RsFileHash& hash) return mFileDatabase->isFileBanned(hash); } +RsFileItem::~RsFileItem() = default; +RsFileItem::RsFileItem(RsFileItemType subtype) : + RsItem( RS_PKT_VERSION_SERVICE, + static_cast(RsServiceType::FILE_TRANSFER), + static_cast(subtype) ) {} + +void RsFileSearchRequestItem::clear() { queryString.clear(); } + +void RsFileSearchResultItem::clear() { mResults.clear(); } diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index a8879541a..0cc88d6aa 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -46,7 +46,8 @@ #include "turtle/turtleclientservice.h" #include "services/p3service.h" #include "retroshare/rsfiles.h" - +#include "rsitems/rsitem.h" +#include "serialiser/rsserial.h" #include "pqi/pqi.h" #include "pqi/p3cfgmgr.h" @@ -67,7 +68,53 @@ class p3PeerMgr; class p3ServiceControl; class p3FileDatabase; -class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService, public RsServiceSerializer +enum class RsFileItemType : uint8_t +{ + NONE = 0x00, /// Only to detect ununitialized + FILE_SEARCH_REQUEST = 0x57, + FILE_SEARCH_RESULT = 0x58 +}; + +struct RsFileItem : RsItem +{ + ~RsFileItem() override; + +protected: + RsFileItem(RsFileItemType subtype); +}; + +struct RsFileSearchRequestItem : RsFileItem +{ + RsFileSearchRequestItem() : RsFileItem(RsFileItemType::FILE_SEARCH_REQUEST) + { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_REQUEST); } + + std::string queryString; + + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { RS_SERIAL_PROCESS(queryString); } + + void clear() override; +}; + +struct RsFileSearchResultItem : RsFileItem +{ + RsFileSearchResultItem() : RsFileItem(RsFileItemType::FILE_SEARCH_RESULT) + { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT); } + + std::vector mResults; + + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { RS_SERIAL_PROCESS(mResults); } + + void clear() override; +}; + + +class ftServer : + public p3Service, public RsFiles, public ftDataSend, + public RsTurtleClientService, public RsServiceSerializer { public: @@ -98,7 +145,21 @@ public: uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; } virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; - virtual void ftReceiveSearchResult(RsTurtleFTSearchResultItem *item); // We dont use TurtleClientService::receiveSearchResult() because of backward compatibility. + + /// We keep this for retro-compatibility @see RsTurtleClientService + virtual void ftReceiveSearchResult(RsTurtleFTSearchResultItem *item); + + /// @see RsTurtleClientService + bool receiveSearchRequest( + unsigned char* searchRequestData, uint32_t searchRequestDataLen, + unsigned char*& search_result_data, uint32_t& searchResultDataLen, + uint32_t& maxAllowsHits ) override; + + /// @see RsTurtleClientService + void receiveSearchResult( + TurtleSearchRequestId requestId, unsigned char* searchResultData, + uint32_t searchResultDataLen ) override; + virtual RsItem *create_item(uint16_t service,uint8_t item_type) const ; virtual RsServiceSerializer *serializer() { return this ; } @@ -148,7 +209,7 @@ public: /// @see RsFiles virtual bool turtleSearchRequest( const std::string& matchString, - const std::function& results)>& multiCallback, + const std::function& results)>& multiCallback, rstime_t maxWait = 300 ); virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ; @@ -337,13 +398,15 @@ private: std::map< TurtleRequestId, std::pair< - std::function& results)>, + std::function& results)>, std::chrono::system_clock::time_point > > mSearchCallbacksMap; RsMutex mSearchCallbacksMapMutex; /// Cleanup mSearchCallbacksMap void cleanTimedOutSearches(); + + RS_SET_CONTEXT_DEBUG_LEVEL(1) }; diff --git a/libretroshare/src/gxs/rsgixs.h b/libretroshare/src/gxs/rsgixs.h index 76581fe4f..a7ab1db82 100644 --- a/libretroshare/src/gxs/rsgixs.h +++ b/libretroshare/src/gxs/rsgixs.h @@ -165,6 +165,8 @@ public: virtual bool getKey(const RsGxsId &id, RsTlvPublicRSAKey& key) = 0; virtual bool getPrivateKey(const RsGxsId &id, RsTlvPrivateRSAKey& key) = 0; // For signing outgoing messages. virtual bool getIdDetails(const RsGxsId& id, RsIdentityDetails& details) = 0 ; // Proxy function so that we get p3Identity info from Gxs + + virtual ~RsGixs(); }; class GixsReputation diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 989ddf305..6a73ac092 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -257,8 +257,8 @@ #include "util/rsmemory.h" #include "util/stacktrace.h" -#ifdef RS_DEEP_SEARCH -# include "deep_search/deep_search.h" +#ifdef RS_DEEP_CHANNEL_INDEX +# include "deep_search/channelsindex.hpp" #endif /*** @@ -5148,13 +5148,13 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st return mGxsNetTunnel->turtleSearchRequest(match_string,this) ; } -#ifndef RS_DEEP_SEARCH +#ifndef RS_DEEP_CHANNEL_INDEX static bool termSearch(const std::string& src, const std::string& substring) { /* always ignore case */ return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() ); } -#endif // ndef RS_DEEP_SEARCH +#endif // ndef RS_DEEP_CHANNEL_INDEX bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map& group_infos) { @@ -5209,11 +5209,11 @@ void RsGxsNetService::receiveTurtleSearchResults( for (const RsGxsGroupSummary& gps : group_infos) { -#ifndef RS_DEEP_SEARCH +#ifndef RS_DEEP_CHANNEL_INDEX /* Only keep groups that are not locally known, and groups that are * not already in the mDistantSearchResults structure. */ if(grpMeta[gps.mGroupId]) continue; -#else // ndef RS_DEEP_SEARCH +#else // ndef RS_DEEP_CHANNEL_INDEX /* When deep search is enabled search results may bring more info * then we already have also about post that are indexed by xapian, * so we don't apply this filter in this case. */ @@ -5302,9 +5302,9 @@ bool RsGxsNetService::search( const std::string& substring, { group_infos.clear(); -#ifdef RS_DEEP_SEARCH - std::vector results; - DeepSearch::search(substring, results); +#ifdef RS_DEEP_CHANNEL_INDEX + std::vector results; + DeepChannelsIndex::search(substring, results); for(auto dsr : results) { @@ -5324,7 +5324,7 @@ bool RsGxsNetService::search( const std::string& substring, if((rit = uQ.find("name")) != uQ.end()) s.mGroupName = rit->second; if((rit = uQ.find("signFlags")) != uQ.end()) - s.mSignFlags = std::stoul(rit->second); + s.mSignFlags = static_cast(std::stoul(rit->second)); if((rit = uQ.find("publishTs")) != uQ.end()) s.mPublishTs = static_cast(std::stoll(rit->second)); if((rit = uQ.find("authorId")) != uQ.end()) @@ -5340,7 +5340,7 @@ bool RsGxsNetService::search( const std::string& substring, } } } -#else // RS_DEEP_SEARCH +#else // RS_DEEP_CHANNEL_INDEX RsGxsGrpMetaTemporaryMap grpMetaMap; { RS_STACK_MUTEX(mNxsMutex) ; @@ -5366,7 +5366,7 @@ bool RsGxsNetService::search( const std::string& substring, group_infos.push_back(s); } -#endif // RS_DEEP_SEARCH +#endif // RS_DEEP_CHANNEL_INDEX #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl; diff --git a/libretroshare/src/gxs/rsgxsutil.cc b/libretroshare/src/gxs/rsgxsutil.cc index 14b93cda8..2396618a8 100644 --- a/libretroshare/src/gxs/rsgxsutil.cc +++ b/libretroshare/src/gxs/rsgxsutil.cc @@ -29,8 +29,8 @@ #include "pqi/pqihash.h" #include "gxs/rsgixs.h" -#ifdef RS_DEEP_SEARCH -# include "deep_search/deep_search.h" +#ifdef RS_DEEP_CHANNEL_INDEX +# include "deep_search/channelsindex.hpp" # include "services/p3gxschannels.h" # include "rsitems/rsgxschannelitems.h" #endif @@ -148,12 +148,12 @@ bool RsGxsMessageCleanUp::clean() RsGxsIntegrityCheck::RsGxsIntegrityCheck( RsGeneralDataService* const dataService, RsGenExchange* genex, RsSerialType& -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX serializer #endif , RsGixs* gixs ) : mDs(dataService), mGenExchangeClient(genex), -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX mSerializer(serializer), #endif mDone(false), mIntegrityMutex("integrity"), mGixs(gixs) {} @@ -168,7 +168,7 @@ void RsGxsIntegrityCheck::run() bool RsGxsIntegrityCheck::check() { -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS; std::set indexedGroups; #endif @@ -221,7 +221,7 @@ bool RsGxsIntegrityCheck::check() } else msgIds.erase(msgIds.find(grp->grpId)); -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX if( isGxsChannels && grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC && grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) @@ -241,7 +241,7 @@ bool RsGxsIntegrityCheck::check() cg.mMeta = meta; indexedGroups.insert(grp->grpId); - DeepSearch::indexChannelGroup(cg); + DeepChannelsIndex::indexChannelGroup(cg); } else { @@ -256,14 +256,15 @@ bool RsGxsIntegrityCheck::check() delete rIt; } -#endif +#endif // def RS_DEEP_CHANNEL_INDEX } else { grpsToDel.push_back(grp->grpId); -#ifdef RS_DEEP_SEARCH - if(isGxsChannels) DeepSearch::removeChannelFromIndex(grp->grpId); -#endif +#ifdef RS_DEEP_CHANNEL_INDEX + if(isGxsChannels) + DeepChannelsIndex::removeChannelFromIndex(grp->grpId); +#endif // def RS_DEEP_CHANNEL_INDEX } if( !(grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) && @@ -320,10 +321,10 @@ bool RsGxsIntegrityCheck::check() if (nxsMsgIt == nxsMsgV.end()) { msgsToDel[grpId].insert(msgId); -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX if(isGxsChannels) - DeepSearch::removeChannelPostFromIndex(grpId, msgId); -#endif + DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId); +#endif // def RS_DEEP_CHANNEL_INDEX } } } @@ -348,14 +349,15 @@ bool RsGxsIntegrityCheck::check() << " with wrong hash or null meta data. meta=" << (void*)msg->metaData << std::endl; msgsToDel[msg->grpId].insert(msg->msgId); -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX if(isGxsChannels) - DeepSearch::removeChannelPostFromIndex(msg->grpId, msg->msgId); -#endif + DeepChannelsIndex::removeChannelPostFromIndex( + msg->grpId, msg->msgId ); +#endif // def RS_DEEP_CHANNEL_INDEX } else if (subscribed_groups.count(msg->metaData->mGroupId)) { -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) ) { @@ -373,7 +375,7 @@ bool RsGxsIntegrityCheck::check() cgIt->toChannelPost(cg, false); cg.mMeta = meta; - DeepSearch::indexChannelPost(cg); + DeepChannelsIndex::indexChannelPost(cg); } else if(dynamic_cast(rIt)) {} else if(dynamic_cast(rIt)) {} @@ -391,7 +393,7 @@ bool RsGxsIntegrityCheck::check() delete rIt; } -#endif +#endif // def RS_DEEP_CHANNEL_INDEX if(!msg->metaData->mAuthorId.isNull()) { diff --git a/libretroshare/src/gxs/rsgxsutil.h b/libretroshare/src/gxs/rsgxsutil.h index c1a610a3d..5acbd1f0b 100644 --- a/libretroshare/src/gxs/rsgxsutil.h +++ b/libretroshare/src/gxs/rsgxsutil.h @@ -213,7 +213,7 @@ private: RsGeneralDataService* const mDs; RsGenExchange *mGenExchangeClient; -#ifdef RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX RsSerialType& mSerializer; #endif bool mDone; diff --git a/libretroshare/src/gxstunnel/p3gxstunnel.cc b/libretroshare/src/gxstunnel/p3gxstunnel.cc index 5f75c633b..0a1a45024 100644 --- a/libretroshare/src/gxstunnel/p3gxstunnel.cc +++ b/libretroshare/src/gxstunnel/p3gxstunnel.cc @@ -1596,7 +1596,13 @@ bool p3GxsTunnelService::getTunnelInfo(const RsGxsTunnelId& tunnel_id,GxsTunnelI // Data packets - info.pending_data_packets = 0; + info.pending_data_packets = 0; + RsPeerId p(tunnel_id); + + for(auto it(pendingGxsTunnelDataItems.begin());it!=pendingGxsTunnelDataItems.end();++it) + if(it->second.data_item->PeerId() == p) + ++info.pending_data_packets ; + info.total_data_packets_sent=0 ; info.total_data_packets_received=0 ; @@ -1704,7 +1710,16 @@ bool p3GxsTunnelService::getTunnelsInfo(std::vectorsecond.status ; ti.total_size_sent = it->second.total_sent ; ti.total_size_received = it->second.total_received ; - + + ti.pending_data_packets = 0; + RsPeerId p(it->first); + for(auto it(pendingGxsTunnelDataItems.begin());it!=pendingGxsTunnelDataItems.end();++it) + if(it->second.data_item->PeerId() == p) + ++ti.pending_data_packets ; + + ti.total_data_packets_sent =0; // not accounted for yet. + ti.total_data_packets_received=0 ; // not accounted for yet. + infos.push_back(ti) ; } diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 574311bb8..01cbc8852 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -404,9 +404,7 @@ HEADERS += pqi/authssl.h \ pqi/pqiqosstreamer.h \ pqi/sslfns.h \ pqi/pqinetstatebox.h \ - pqi/p3servicecontrol.h \ - -# pqi/p3dhtmgr.h \ + pqi/p3servicecontrol.h HEADERS += rsserver/p3face.h \ rsserver/p3history.h \ @@ -653,8 +651,8 @@ equals(RS_UPNP_LIB, miniupnpc) { contains(RS_UPNP_LIB, upnp) { HEADERS += rs_upnp/upnp18_retrocompat.h - HEADERS += rs_upnp/UPnPBase.h rs_upnp/upnphandler_linux.h - SOURCES += rs_upnp/UPnPBase.cpp rs_upnp/upnphandler_linux.cc + HEADERS += rs_upnp/UPnPBase.h rs_upnp/upnphandler_libupnp.h + SOURCES += rs_upnp/UPnPBase.cpp rs_upnp/upnphandler_libupnp.cc } # new gxs cache system @@ -901,8 +899,32 @@ rs_jsonapi { SOURCES += jsonapi/jsonapi.cpp } -rs_deep_search { - HEADERS += deep_search/deep_search.h +rs_deep_channels_index { + HEADERS *= deep_search/commonutils.hpp + SOURCES *= deep_search/commonutils.cpp + + HEADERS += deep_search/channelsindex.hpp + SOURCES += deep_search/channelsindex.cpp +} + +rs_deep_files_index { + HEADERS *= deep_search/commonutils.hpp + SOURCES *= deep_search/commonutils.cpp + + HEADERS += deep_search/filesindex.hpp + SOURCES += deep_search/filesindex.cpp +} + +rs_deep_files_index_ogg { + HEADERS += deep_search/filesoggindexer.hpp +} + +rs_deep_files_index_flac { + HEADERS += deep_search/filesflacindexer.hpp +} + +rs_deep_files_index_taglib { + HEADERS += deep_search/filestaglibindexer.hpp } rs_broadcast_discovery { diff --git a/libretroshare/src/pqi/p3dhtmgr.cc b/libretroshare/src/pqi/p3dhtmgr.cc deleted file mode 100644 index d54a1cbf2..000000000 --- a/libretroshare/src/pqi/p3dhtmgr.cc +++ /dev/null @@ -1,1795 +0,0 @@ -/******************************************************************************* - * libretroshare/src/pqi: p3dhtmgr.cc * - * * - * libretroshare: retroshare core library * - * * - * Copyright 2004-2008 by Robert Fernie. * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public License * - * along with this program. If not, see . * - * * - *******************************************************************************/ - -#include -#include -#include -#include -#include "util/rstime.h" - -#include "pqi/p3dhtmgr.h" -#include "pqi/p3peermgr.h" -#include "pqi/p3linkmgr.h" - -#include "util/rsprint.h" -#include "util/rsdebug.h" -#include "util/rsstring.h" -const int p3dhtzone = 3892; - -/***** - * #define DHT_DEBUG 1 - * #define DHT_LOG 1 // FOR LOGGING DHT STUFF. - * #define P3DHTMGR_USE_LOCAL_UDP_CONN 1 // For Testing only - ****/ - -/**** DHT State Variables **** - * TODO: - * (1) notify call in. - * (2) publish/search parameters. - * (3) callback. - * (4) example - * - */ - -/**** DHT State Variables ****/ - -#define DHT_STATE_OFF 0 -#define DHT_STATE_INIT 1 -#define DHT_STATE_CHECK_PEERS 2 -#define DHT_STATE_FIND_STUN 3 -#define DHT_STATE_ACTIVE 4 - -/* TIMEOUTS (Key ones in .h) */ -#define DHT_RESTART_PERIOD 300 /* 5 min */ -#define DHT_DEFAULT_PERIOD 300 /* Default period if no work to do */ -#define DHT_MIN_PERIOD 1 /* to ensure we don't get too many requests */ -#define DHT_SHORT_PERIOD 10 /* a short period */ - -#define DHT_DEFAULT_WAITTIME 1 /* Std sleep break period */ - -#define DHT_NUM_BOOTSTRAP_BINS 8 -#define DHT_MIN_BOOTSTRAP_REQ_PERIOD 5 - -void printDhtPeerEntry(dhtPeerEntry *ent, std::ostream &out); - -/* Interface class for DHT data */ - -dhtPeerEntry::dhtPeerEntry() - :state(DHT_PEER_INIT), lastTS(0), - notifyPending(0), notifyTS(0), - type(DHT_ADDR_INVALID) -{ - laddr.sin_addr.s_addr = 0; - laddr.sin_port = 0; - laddr.sin_family = 0; - - raddr.sin_addr.s_addr = 0; - raddr.sin_port = 0; - raddr.sin_family = 0; - - return; -} - -p3DhtMgr::p3DhtMgr(RsPeerId id, pqiConnectCb *cb) - :pqiNetAssistConnect(id, cb), dhtMtx("p3DhtMgr"), mStunRequired(true) -{ - /* setup own entry */ - dhtMtx.lock(); /* LOCK MUTEX */ - - ownEntry.id = id; - ownEntry.state = DHT_PEER_INIT; - ownEntry.type = DHT_ADDR_INVALID; - ownEntry.lastTS = 0; - - ownEntry.notifyPending = 0; - ownEntry.notifyTS = 0; - - ownEntry.hash1 = RsUtil::HashId(id, false); - ownEntry.hash2 = RsUtil::HashId(id, true); - - mDhtModifications = true; - mDhtOn = false; - mDhtState = DHT_STATE_OFF; - - mBootstrapAllowed = true; - mLastBootstrapListTS = 0; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return; -} - - /* OVERLOADED from pqiNetAssistConnect - */ - -void p3DhtMgr::shutdown() -{ - /* ??? */ - -} - -void p3DhtMgr::restart() -{ - /* ??? */ -} - -void p3DhtMgr::enable(bool on) -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - mDhtModifications = true; - mDhtOn = on; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ -} - -bool p3DhtMgr::getEnabled() -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - bool on = mDhtOn; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return on; -} - -bool p3DhtMgr::getActive() -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - bool act = dhtActive(); - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return act; -} - -void p3DhtMgr::setBootstrapAllowed(bool on) -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - mBootstrapAllowed = on; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ -} - -bool p3DhtMgr::getBootstrapAllowed() -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - bool on = mBootstrapAllowed; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return on; -} - -/******************************** PEER MANAGEMENT ********************************** - * - */ - /* set key data */ -bool p3DhtMgr::setExternalInterface( - struct sockaddr_in laddr, - struct sockaddr_in raddr, - uint32_t type) -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - mDhtModifications = true; - ownEntry.laddr = laddr; - ownEntry.raddr = raddr; - ownEntry.type = type; - ownEntry.state = DHT_PEER_ADDR_KNOWN; /* will force republish */ - ownEntry.lastTS = 0; /* will force republish */ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::setExternalInterface()"; - std::cerr << " laddr: " << rs_inet_ntoa(ownEntry.laddr.sin_addr); - std::cerr << " lport: " << ntohs(ownEntry.laddr.sin_port); - std::cerr << " raddr: " << rs_inet_ntoa(ownEntry.raddr.sin_addr); - std::cerr << " rport: " << ntohs(ownEntry.raddr.sin_port); - std::cerr << " type: " << ownEntry.type; - std::cerr << " state: " << ownEntry.state; - std::cerr << std::endl; -#endif - -#ifdef DHT_LOGS - /* Log External Interface too */ - std::string out = "p3DhtMgr::setExternalInterface()"; - out += " laddr: " + rs_inet_ntoa(ownEntry.laddr.sin_addr); - rs_sprintf_append(out, " lport: %u", ntohs(ownEntry.laddr.sin_port)); - out += " raddr: " + rs_inet_ntoa(ownEntry.raddr.sin_addr); - rs_sprintf_append(out, " rport: %u", ntohs(ownEntry.raddr.sin_port)); - rs_sprintf_append(out, " type: %lu", ownEntry.type); - rs_sprintf_append(out, " state: %lu", ownEntry.state); - - rslog(RSL_WARNING, p3dhtzone, out); -#endif - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - checkOwnDHTKeys(); - return true; -} - - - /* add / remove peers */ -bool p3DhtMgr::findPeer(const RsPeerId& id) -{ - RsStackMutex stack(dhtMtx); /***** LOCK MUTEX *****/ - - mDhtModifications = true; - - std::map::iterator it; - it = peers.find(id); - if (it != peers.end()) - { - /* reset some of it */ - it->second.state = DHT_PEER_INIT; - - // No point destroying a valid address!. - //it->second.type = DHT_ADDR_INVALID; - - // Reset notify - it->second.notifyPending = 0; - it->second.notifyTS = 0; - - return true; - } - - /* if they are not in the list -> add */ - dhtPeerEntry ent; - ent.id = id; - ent.state = DHT_PEER_INIT; - ent.type = DHT_ADDR_INVALID; - ent.lastTS = 0; - - ent.notifyPending = 0; - ent.notifyTS = 0; - - /* fill in hashes */ - ent.hash1 = RsUtil::HashId(id, false); - ent.hash2 = RsUtil::HashId(id, true); - - /* other fields don't matter */ - - peers[id] = ent; - - return true; -} - -bool p3DhtMgr::dropPeer(const RsPeerId& id) -{ - RsStackMutex stack(dhtMtx); /***** LOCK MUTEX *****/ - - mDhtModifications = true; - - /* once we are connected ... don't worry about them anymore */ - std::map::iterator it; - it = peers.find(id); - if (it == peers.end()) - { - return false; - } - - /* leave it in there - just switch to off */ - it->second.state = DHT_PEER_OFF; - - return true; -} - - /* post DHT key saying we should connect */ -bool p3DhtMgr::notifyPeer(const RsPeerId& id) -{ - RsStackMutex stack(dhtMtx); /***** LOCK MUTEX *****/ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::notifyPeer() " << id.toStdString() << std::endl; -#endif - - std::map::iterator it; - it = peers.find(id); - if (it == peers.end()) - { - return false; - } - /* ignore OFF peers */ - if (it->second.state == DHT_PEER_OFF) - { - return false; - } - - - rstime_t now = time(NULL); - - if (now - it->second.notifyTS < 2 * DHT_NOTIFY_PERIOD) - { - /* drop the notify (too soon) */ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::notifyPeer() TO SOON - DROPPING" << std::endl; -#endif -#ifdef DHT_LOGS - { - /* Log */ - rslog(RSL_WARNING, p3dhtzone, "p3DhtMgr::notifyPeer() Id: " + id.toStdString() + " TO SOON - DROPPING"); - - } -#endif - return false; - } - - it->second.notifyPending = RS_CONNECT_ACTIVE; - it->second.notifyTS = time(NULL); - - /* Trigger search if not found! */ - if (it->second.state != DHT_PEER_FOUND) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::notifyPeer() PEER NOT FOUND - Trigger search" << std::endl; -#endif -#ifdef DHT_LOGS - { - /* Log */ - rslog(RSL_WARNING, p3dhtzone, "p3DhtMgr::notifyPeer() Id: " + id.toStdString() + " PEER NOT FOUND - Trigger Search"); - } -#endif - it->second.lastTS = 0; - } - - mDhtModifications = true; /* no wait! */ - - return true; -} - - /* extract current peer status */ -bool p3DhtMgr::getPeerStatus(const RsPeerId &id, - struct sockaddr_storage &laddr, - struct sockaddr_storage &raddr, - uint32_t &type, uint32_t &state) -{ - RsStackMutex stack(dhtMtx); /* LOCK MUTEX */ - - std::map::iterator it; - it = peers.find(id); - - /* ignore OFF peers */ - if ((it == peers.end()) || (it->second.state == DHT_PEER_OFF)) - { - return false; - } - - laddr = it->second.laddr; - raddr = it->second.raddr; - type = it->second.type; - state = it->second.type; - - return true; -} - -/********************************* STUN HANDLING ********************************** - * add / cleanup / stun. - * - */ - - /* stun */ -bool p3DhtMgr::addStun(std::string id) -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - mDhtModifications = true; - - std::list::iterator it; - it = std::find(stunIds.begin(), stunIds.end(), id); - if (it != stunIds.end()) - { - dhtMtx.unlock(); /* UNLOCK MUTEX */ - return false; - } - stunIds.push_back(id); - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return true; -} - -bool p3DhtMgr::enableStun(bool on) -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - mDhtModifications = true; - - if (!on) - { - /* clear up */ - stunIds.clear(); - } - - mStunRequired = on; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return true; -} - - -void p3DhtMgr::run() -{ - /* - * - */ - - while(1) - { - checkDHTStatus(); - - -#ifdef DHT_DEBUG - status(std::cerr); -#endif - - dhtMtx.lock(); /* LOCK MUTEX */ - - uint32_t dhtState = mDhtState; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - int period = 60; /* default wait */ - switch(dhtState) - { - case DHT_STATE_INIT: -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() state = INIT -> wait" << std::endl; -#endif - period = 10; - break; - case DHT_STATE_CHECK_PEERS: -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() state = CHECK_PEERS -> do stuff" << std::endl; -#endif - checkPeerDHTKeys(); - checkStunState(); - period = DHT_MIN_PERIOD; - break; - case DHT_STATE_FIND_STUN: -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() state = FIND_STUN -> do stuff" << std::endl; -#endif - doStun(); - checkPeerDHTKeys(); /* keep on going - as we could be in this state for a while */ - checkStunState(); - period = DHT_MIN_PERIOD; - break; - case DHT_STATE_ACTIVE: - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() state = ACTIVE -> do stuff" << std::endl; -#endif - - period = checkOwnDHTKeys(); -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() checkOwnDHTKeys() period: " << period << std::endl; -#endif - int tmpperiod = checkNotifyDHT(); -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() checkNotifyDHT() period: " << tmpperiod << std::endl; -#endif - int tmpperiod2 = checkPeerDHTKeys(); -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() checkPeerDHTKeys() period: " << tmpperiod2 << std::endl; -#endif - if (tmpperiod < period) - period = tmpperiod; - if (tmpperiod2 < period) - period = tmpperiod2; - - /* finally we need to keep stun going */ - if (checkStunState_Active()) - { - /* still more stun to do */ - period = DHT_SHORT_PERIOD; - doStun(); - } - - } - break; - default: - case DHT_STATE_OFF: -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() state = OFF -> wait" << std::endl; -#endif - period = 60; - break; - } - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() sleeping for: " << period << std::endl; -#endif - - /* Breakable sleep loop */ - - bool toBreak = false; - int waittime = 1; - int i; - for(i = 0; i < period; i += waittime) - { - if (period-i > DHT_DEFAULT_WAITTIME) - { - waittime = DHT_DEFAULT_WAITTIME; - } - else - { - waittime = period-i; - } - - dhtMtx.lock(); /* LOCK MUTEX */ - - if (mDhtModifications) - { - mDhtModifications = false; - toBreak = true; - } - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - if (toBreak) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::run() breaking sleep" << std::endl; -#endif - - break; /* speed up config modifications */ - } - -/********************************** WINDOWS/UNIX SPECIFIC PART ******************/ -#ifndef WINDOWS_SYS - sleep(waittime); -#else - Sleep(1000 * waittime); -#endif -/********************************** WINDOWS/UNIX SPECIFIC PART ******************/ - - } - } -} - - -int p3DhtMgr::checkOwnDHTKeys() -{ - int repubPeriod = 10000; - rstime_t now = time(NULL); - - /* in order of importance: - * (1) Check for Own Key publish. - * (2) Check for notification requests - * (3) Check for Peers - */ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys()" << std::endl; -#endif - - dhtMtx.lock(); /* LOCK MUTEX */ - - dhtPeerEntry peer = ownEntry; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - /* publish ourselves if necessary */ - if (peer.state >= DHT_PEER_ADDR_KNOWN) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() OWN ADDR KNOWN" << std::endl; -#endif - if ((peer.state < DHT_PEER_PUBLISHED) || - (now - peer.lastTS > DHT_PUBLISH_PERIOD)) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() OWN ADDR REPUB" << std::endl; -#endif - -#ifdef DHT_DEBUG - std::cerr << "PUBLISH: "; - std::cerr << " hash1: " << RsUtil::BinToHex(peer.hash1); - std::cerr << " laddr: " << rs_inet_ntoa(peer.laddr.sin_addr); - std::cerr << ":" << ntohs(peer.laddr.sin_port); - std::cerr << " raddr: " << rs_inet_ntoa(peer.raddr.sin_addr); - std::cerr << ":" << ntohs(peer.raddr.sin_port); - std::cerr << " type: " << peer.type; - std::cerr << std::endl; -#endif - -#ifdef DHT_LOGS - { - /* Log */ - std::string out = "p3DhtMgr::checkOwnDHTKeys() PUBLISH OWN ADDR:"; - out += " hash1: " + RsUtil::BinToHex(peer.hash1); - out += " laddr: " + rs_inet_ntoa(peer.laddr.sin_addr); - rs_sprintf_append(out, ":%u", ntohs(peer.laddr.sin_port)); - out += " raddr: " + rs_inet_ntoa(peer.raddr.sin_addr); - rs_sprintf_append(out, ":%u", ntohs(peer.raddr.sin_port)); - rs_sprintf_append(out, " type: %lu", peer.type); - - rslog(RSL_WARNING, p3dhtzone, out); - } -#endif - - /* publish own key */ - if (dhtPublish(peer.hash1, peer.laddr, peer.raddr, peer.type, "")) - { - dhtMtx.lock(); /* LOCK MUTEX */ - - ownEntry.lastTS = now; - ownEntry.state = DHT_PEER_PUBLISHED; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - } - - /* dhtBootstrap -> if allowed and EXT port */ - if (peer.type & RS_NET_CONN_TCP_EXTERNAL) - { - dhtMtx.lock(); /* LOCK MUTEX */ - - bool doBootstrapPub = mBootstrapAllowed; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - if (doBootstrapPub) - { - dhtBootstrap(randomBootstrapId(), peer.hash1, ""); - } - } - - /* restart immediately */ - repubPeriod = DHT_MIN_PERIOD; - return repubPeriod; - } - else - { - if (now - peer.lastTS < DHT_PUBLISH_PERIOD) - { - repubPeriod = DHT_PUBLISH_PERIOD - - (now - peer.lastTS); - } - else - { - repubPeriod = 10; - } -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() repub in: "; - std::cerr << repubPeriod << std::endl; -#endif - } - - - /* check for connect requests */ - //if ((peer.state == DHT_PEER_PUBLISHED) && - // (!(peer.type & RS_NET_CONN_TCP_EXTERNAL))) - if (peer.state == DHT_PEER_PUBLISHED) - { - if (now - peer.notifyTS >= DHT_NOTIFY_PERIOD) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() check for Notify (rep=0)"; - std::cerr << std::endl; -#endif -#ifdef DHT_LOGS - { - /* Log */ - rslog(RSL_WARNING, p3dhtzone, "p3DhtMgr::checkOwnDHTKeys() Checking for NOTIFY"); - } -#endif - - if (dhtSearch(peer.hash1, DHT_MODE_NOTIFY)) - { - dhtMtx.lock(); /* LOCK MUTEX */ - - ownEntry.notifyTS = now; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - } - - /* restart immediately */ - repubPeriod = DHT_MIN_PERIOD; - return repubPeriod; - } - else - { - repubPeriod = DHT_NOTIFY_PERIOD - - (now - peer.notifyTS); - if (repubPeriod < DHT_MIN_PERIOD) - { - repubPeriod = DHT_MIN_PERIOD; - } -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() check Notify in: "; - std::cerr << repubPeriod << std::endl; -#endif - } - } - else - { - if (peer.state != DHT_PEER_PUBLISHED) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() No Notify until Published"; - std::cerr << std::endl; -#endif - } - else if (peer.type & RS_NET_CONN_TCP_EXTERNAL) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() No Notify because have Ext Addr"; - std::cerr << std::endl; -#endif - } - else - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() No Notify: Unknown Reason"; - std::cerr << std::endl; -#endif - } - } - - } - else - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkOwnDHTKeys() PEER ADDR UNKNOWN" << std::endl; -#endif - repubPeriod = 10; - } - return repubPeriod; -} - - -int p3DhtMgr::checkPeerDHTKeys() -{ - /* now loop through the peers */ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkPeerDHTKeys()" << std::endl; -#endif - - dhtMtx.lock(); /* LOCK MUTEX */ - - /* iterate through and find min time and suitable candidate */ - std::map::iterator it,pit; - rstime_t now = time(NULL); - uint32_t period = 0; - uint32_t repeatPeriod = 6000; - - pit = peers.end(); - rstime_t pTS = now; - - for(it = peers.begin(); it != peers.end(); it++) - { - /* ignore OFF peers */ - if (it->second.state == DHT_PEER_OFF) - { - continue; - } - - rstime_t delta = now - it->second.lastTS; - if (it->second.state < DHT_PEER_FOUND) - { - period = DHT_SEARCH_PERIOD; - } - else /* (it->second.state == DHT_PEER_FOUND) */ - { - period = DHT_CHECK_PERIOD; - } -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkPeerDHTKeys() Peer: " << it->second.id.toStdString(); - std::cerr << " Period: " << period; - std::cerr << " Delta: " << delta; - std::cerr << std::endl; -#endif - - - if ((unsigned) delta >= period) - { - if (it->second.lastTS < pTS) - { - pit = it; - pTS = it->second.lastTS; - } - repeatPeriod = DHT_MIN_PERIOD; - } - else if (period - delta < repeatPeriod) - { - repeatPeriod = period - delta; - } - } - - /* now have - peer to handle, and period to next call */ - - if (pit == peers.end()) - { - dhtMtx.unlock(); /* UNLOCK MUTEX */ - return repeatPeriod; - } - - /* update timestamp - * clear FOUND or INIT state. - * */ - - pit->second.lastTS = now; - pit->second.state = DHT_PEER_SEARCH; - - dhtPeerEntry peer = (pit->second); - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - /* now search for the peer */ - dhtSearch(peer.hash1, DHT_MODE_SEARCH); - - /* results come through callback */ - return repeatPeriod; -} - - -int p3DhtMgr::checkNotifyDHT() -{ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkNotifyDHT()" << std::endl; -#endif - /* now loop through the peers */ - uint32_t notifyType = 0; - dhtPeerEntry peer; - dhtPeerEntry own; - - { - RsStackMutex stack(dhtMtx); /***** LOCK MUTEX *****/ - - /* iterate through and find min time and suitable candidate */ - std::map::iterator it; - rstime_t now = time(NULL); - int repeatPeriod = DHT_DEFAULT_PERIOD; - - /* find the first with a notify flag */ - for(it = peers.begin(); it != peers.end(); it++) - { - /* ignore OFF peers */ - if (it->second.state == DHT_PEER_OFF) - { - continue; - } - - if (it->second.notifyPending) - { - /* if very old drop it */ - if (now - it->second.notifyTS > DHT_NOTIFY_PERIOD) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkNotifyDHT() Dropping OLD Notify: "; - std::cerr << it->first << std::endl; -#endif - it->second.notifyPending = 0; - } - - if (it->second.state == DHT_PEER_FOUND) - { - notifyType = it->second.notifyPending; - break; - } - } - } - - /* now have - peer to handle */ - if (it == peers.end()) - { - return repeatPeriod; - } - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkNotifyDHT() Notify From: "; - std::cerr << it->first << std::endl; -#endif -#ifdef DHT_LOGS - { - /* Log */ - rslog(RSL_WARNING, p3dhtzone, "p3DhtMgr::checkNotifyDHT() Notify Request for Known Peer: " + it->first); - } -#endif - - /* update timestamp */ - it->second.notifyTS = now; - it->second.notifyPending = 0; - - peer = (it->second); - own = ownEntry; - - } /******* UNLOCK ******/ - - if (notifyType == RS_CONNECT_ACTIVE) - { - /* publish notification (publish Our Id) - * We publish the connection attempt on peers hash, - * using our alternative hash.. - * */ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkNotifyDHT() Posting Active Notify"; - std::cerr << std::endl; -#endif -#ifdef DHT_LOGS - { - /* Log */ - rslog(RSL_WARNING, p3dhtzone, "p3DhtMgr::checkNotifyDHT() POST DHT (Active Notify) for Peer: " + peer.id); - } -#endif - - dhtNotify(peer.hash1, own.hash2, ""); - } - - /* feedback to say we started it! */ -#ifdef P3DHTMGR_USE_LOCAL_UDP_CONN - mConnCb->peerConnectRequest(peer.id, peer.laddr,peer.laddr,peer.laddr, RS_CB_DHT, 0, 0,0); -#else - mConnCb->peerConnectRequest(peer.id, peer.raddr,peer.laddr,peer.laddr, RS_CB_DHT, 0, 0, 0); -#endif - - return DHT_MIN_PERIOD; -} - - - -int p3DhtMgr::doStun() -{ - if (stunIds.size() < 1) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::doStun() Failed -> no Ids left" << std::endl; -#endif - return 0; - } - - dhtMtx.lock(); /* LOCK MUTEX */ - - bool stunRequired = mStunRequired; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - /* now loop through the peers */ - if (!stunRequired) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::doStun() not Required" << std::endl; -#endif - return 0; - } - - /* pop the front one */ - std::string activeStunId = stunIds.front(); - - stunIds.pop_front(); - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::doStun() STUN: " << RsUtil::BinToHex(activeStunId); - std::cerr << std::endl; -#endif - - /* look it up! */ - dhtSearch(activeStunId, DHT_MODE_SEARCH); - - return 1; -} - - - -int p3DhtMgr::checkStunState() -{ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkStunState()" << std::endl; -#endif - dhtMtx.lock(); /* LOCK MUTEX */ - - /* now loop through the peers */ - if (!mStunRequired) - { - mDhtState = DHT_STATE_ACTIVE; - } - - if (mDhtState == DHT_STATE_CHECK_PEERS) - { - /* check that they have all be searched for */ - std::map::iterator it; - for(it = peers.begin(); it != peers.end(); it++) - { - if (it->second.state == DHT_PEER_INIT) - { - break; - } - } - - if (it == peers.end()) - { - /* we've checked all peers */ - mDhtState = DHT_STATE_FIND_STUN; - } - } - else if (mDhtState == DHT_STATE_FIND_STUN) - { - } - - /* if we run out of stun peers -> get some more */ - if (stunIds.size() < 1) - { -#ifdef DHT_DEBUG - std::cerr << "WARNING: out of Stun Peers - Fetching some more" << std::endl; -#endif - mDhtState = DHT_STATE_ACTIVE; - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - /* this is a locked function */ - getDhtBootstrapList(); - - dhtMtx.lock(); /* LOCK MUTEX */ - } - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - return 1; -} - -int p3DhtMgr::checkStunState_Active() -{ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkStunState_Active()" << std::endl; -#endif - dhtMtx.lock(); /* LOCK MUTEX */ - - bool stunReq = mStunRequired; - bool moreIds = ((mStunRequired) && (stunIds.size() < 1)); - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - if (moreIds) - /* if we run out of stun peers -> get some more */ - { -#ifdef DHT_DEBUG - std::cerr << "WARNING: out of Stun Peers - getting more" << std::endl; -#endif - getDhtBootstrapList(); - } - - return stunReq; -} - -bool p3DhtMgr::getDhtBootstrapList() -{ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::getDHTBootstrapList()" << std::endl; -#endif - dhtMtx.lock(); /* LOCK MUTEX */ - - rstime_t now = time(NULL); - if (now - mLastBootstrapListTS < DHT_MIN_BOOTSTRAP_REQ_PERIOD) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::getDHTBootstrapList() Waiting: "; - std::cerr << DHT_MIN_BOOTSTRAP_REQ_PERIOD-(now-mLastBootstrapListTS); - std::cerr << " secs" << std::endl; -#endif - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return false; - } - - mLastBootstrapListTS = now; - std::string bootId = randomBootstrapId(); - - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::getDHTBootstrapList() bootId: 0x"; - std::cerr << RsUtil::BinToHex(bootId) << std::endl; -#endif - - dhtSearch(bootId, DHT_MODE_SEARCH); - - return true; -} - - -std::string p3DhtMgr::BootstrapId(uint32_t bin) -{ - /* generate these from an equation! (makes it easy) - * Make sure that NUM_BOOTSTRAP_BINS doesn't affect ids - */ - - uint32_t id = (bin % DHT_NUM_BOOTSTRAP_BINS) * 1234; - - std::string genId; - rs_sprintf(genId, "BootstrapId%lu", id); - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::BootstrapId() generatedId: "; - std::cerr << genId << std::endl; -#endif - - /* now hash this to create a bootstrap Bin Id */ - std::string bootId = RsUtil::HashId(genId, false); - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::BootstrapId() bootId: 0x"; - std::cerr << RsUtil::BinToHex(bootId) << std::endl; -#endif - - return bootId; -} - -std::string p3DhtMgr::randomBootstrapId() -{ - uint32_t rnd = DHT_NUM_BOOTSTRAP_BINS * (rand() / (RAND_MAX + 1.0)); - - return BootstrapId(rnd); -} - - - -void p3DhtMgr::checkDHTStatus() -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - bool isActive = (mDhtState != DHT_STATE_OFF); - - bool toShutdown = false; - bool toStartup = false; - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() mDhtState: " << mDhtState << std::endl; - std::cerr << "p3DhtMgr::checkDhtStatus() mDhtOn : " << mDhtOn << std::endl; -#endif - - if ((isActive) && (!mDhtOn)) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() Active & !mDhtOn -> toShutdown" << std::endl; -#endif - toShutdown = true; - } - - if ((!isActive) && (mDhtOn)) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() !Active & mDhtOn -> toStartup" << std::endl; -#endif - toStartup = true; - } - - /* restart if it has shutdown */ - if (isActive && mDhtOn) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() Active & mDhtOn" << std::endl; -#endif - if (dhtActive()) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() dhtActive() = true" << std::endl; -#endif - if (mDhtState == DHT_STATE_INIT) - { - mDhtState = DHT_STATE_CHECK_PEERS; -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() Switching to CHECK PEERS" << std::endl; -#endif - } - } - else - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() dhtActive() = false" << std::endl; -#endif - if (mDhtActiveTS - time(NULL) > DHT_RESTART_PERIOD) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() restart Period..." << std::endl; -#endif - toShutdown = true; - toStartup = true; - } - } - } - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - if (toShutdown) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() toShutdown = true -> shutdown()" << std::endl; -#endif - if (dhtShutdown()) - { - clearDhtData(); - - dhtMtx.lock(); /* LOCK MUTEX */ - - mDhtState = DHT_STATE_OFF; -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() mDhtState -> OFF" << std::endl; -#endif - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - } - } - - if (toStartup) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() toStartup = true -> dhtInit()" << std::endl; -#endif - if (dhtInit()) - { - - dhtMtx.lock(); /* LOCK MUTEX */ - - mDhtState = DHT_STATE_INIT; - mDhtActiveTS = time(NULL); - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::checkDhtStatus() mDhtState -> INIT" << std::endl; -#endif - - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - } - } -} - -void p3DhtMgr::clearDhtData() -{ - std::cerr << "p3DhtMgr::clearDhtData() DUMMY FN" << std::endl; -} - - -/****************************** REAL DHT INTERFACE ********************************* - * publish/search/result. - * - * dummy implementation for testing. - */ - -int p3DhtMgr::status(std::ostream &out) -{ - dhtMtx.lock(); /* LOCK MUTEX */ - - out << "p3DhtMgr::status() ************************************" << std::endl; - out << "mDhtState: " << mDhtState << std::endl; - out << "mDhtOn : " << mDhtOn << std::endl; - out << "dhtActive: " << dhtActive() << std::endl; - - /* now own state */ - out << "OWN DETAILS -------------------------------------------" << std::endl; - printDhtPeerEntry(&ownEntry, out); - out << "OWN DETAILS END----------------------------------------" << std::endl; - - /* now peers states */ - std::map::iterator it; - out << "PEER DETAILS ------------------------------------------" << std::endl; - for(it = peers.begin(); it != peers.end(); it++) - { - printDhtPeerEntry(&(it->second), out); - } - out << "PEER DETAILS END---------------------------------------" << std::endl; - - /* now stun states */ - out << "STUN DETAILS ------------------------------------------" << std::endl; - out << "Available Stun Ids: " << stunIds.size() << std::endl; - out << "STUN DETAILS END---------------------------------------" << std::endl; - - - out << "p3DhtMgr::status() END ********************************" << std::endl; - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - return 0; -} - - - - - -bool p3DhtMgr::dhtInit() -{ - std::cerr << "p3DhtMgr::dhtInit() DUMMY FN" << std::endl; - return true; -} - -bool p3DhtMgr::dhtShutdown() -{ - std::cerr << "p3DhtMgr::dhtShutdown() DUMMY FN" << std::endl; - return true; -} - -bool p3DhtMgr::dhtActive() -{ - std::cerr << "p3DhtMgr::dhtActive() DUMMY FN" << std::endl; - return true; -} - -bool p3DhtMgr::publishDHT(std::string /*key*/, std::string /*value*/, uint32_t /*ttl*/) -{ - std::cerr << "p3DhtMgr::publishDHT() DUMMY FN" << std::endl; - return false; -} - -bool p3DhtMgr::searchDHT(std::string /*idhash*/) -{ - std::cerr << "p3DhtMgr::searchDHT() DUMMY FN" << std::endl; - return false; -} - - - - -/****************************** INTERMEDIATE DHT INTERFACE ********************************* - * publish/search/result. - * - * Take the 'real' parameters and create the key/value parameters for the real dht. - */ - - -bool p3DhtMgr::dhtPublish(std::string idhash, - struct sockaddr_in &laddr, struct sockaddr_in &raddr, - uint32_t type, std::string sign) -{ - /* remove unused parameter warnings */ - (void) sign; - - /* ... goes and searches */ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtPublish()" << std::endl; - - std::cerr << "PUBLISHing: idhash: " << RsUtil::BinToHex(idhash); - std::cerr << " laddr: " << rs_inet_ntoa(laddr.sin_addr); - std::cerr << ":" << ntohs(laddr.sin_port); - std::cerr << " raddr: " << rs_inet_ntoa(raddr.sin_addr); - std::cerr << ":" << ntohs(raddr.sin_port); - std::cerr << " type: " << type; - std::cerr << " sign: " << sign; - std::cerr << std::endl; -#endif - - /* Create a Value from addresses and type */ - /* to store the ip address and flags */ - - std::string value; - rs_sprintf(value, "RSDHT:%02d: ", DHT_MODE_SEARCH); - rs_sprintf_append(value, "IPL=%s:%u, ", rs_inet_ntoa(laddr.sin_addr).c_str(), ntohs(laddr.sin_port)); - rs_sprintf_append(value, "IPE=%s:%u, ", rs_inet_ntoa(raddr.sin_addr).c_str(), ntohs(raddr.sin_port)); - rs_sprintf_append(value, "type=%04x, ", type); - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtPublish()" << std::endl; - - std::cerr << "PUBLISH: key: " << RsUtil::BinToHex(idhash); - std::cerr << " value: " << value; - std::cerr << std::endl; -#endif - - /* call to the real DHT */ - return publishDHT(idhash, value, DHT_TTL_PUBLISH); -} - -bool p3DhtMgr::dhtNotify(std::string idhash, std::string ownIdHash, std::string /*sign*/) -{ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtNotify()" << std::endl; -#endif - - std::string value; - rs_sprintf(value, "RSDHT:%02d:%s", DHT_MODE_NOTIFY, ownIdHash.c_str()); - - /* call to the real DHT */ - return publishDHT(idhash, value, DHT_TTL_NOTIFY); -} - -bool p3DhtMgr::dhtSearch(std::string idhash, uint32_t /*mode*/) -{ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtSearch()" << std::endl; -#endif - - /* call to the real DHT */ - return searchDHT(idhash); -} - - -bool p3DhtMgr::dhtBootstrap(std::string storehash, std::string ownIdHash, std::string /*sign*/) -{ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtBootstrap()" << std::endl; -#endif - - std::string value; - rs_sprintf(value, "RSDHT:%02d:%s", DHT_MODE_BOOTSTRAP, ownIdHash.c_str()); - - /* call to the real DHT */ - return publishDHT(storehash, value, DHT_TTL_BOOTSTRAP); -} - - -/****************************** DHT FEEDBACK INTERFACE ********************************* - * Two functions... - * (1) The interpretation function. - * (2) callback handling. - * - */ - -bool p3DhtMgr::resultDHT(std::string key, std::string value) -{ - /* so .... two possibilities. - * - * RSDHT:01: IPL ... IPE ... TYPE ... (advertising) - * RSDHT:02: HASH (connect request) - * - */ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() key: 0x" << RsUtil::BinToHex(key); - std::cerr << " value: 0x" << RsUtil::BinToHex(value) << std::endl; -#endif - - /* variables for dhtResult() call */ - struct sockaddr_in laddr; - struct sockaddr_in raddr; - std::string sign; - - - int32_t reqType; - uint32_t loc; - if (1 > sscanf(value.c_str(), "RSDHT:%d: %n", &reqType, &loc)) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() Not RSDHT msg -> discarding" << std::endl; -#endif - /* failed */ - return false; - } - - - dhtMtx.lock(); /* LOCK MUTEX */ - std::string ownhash = ownEntry.hash1; - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - switch(reqType) - { - case DHT_MODE_NOTIFY: - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() NOTIFY msg" << std::endl; -#endif - - if (ownhash != key) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() NOTIFY msg not for us -> discarding" << std::endl; -#endif - return false; - } - - /* get the hash */ - std::string notifyHash = value.substr(loc); -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() NOTIFY msg HASH:-> 0x" << RsUtil::BinToHex(notifyHash) << "<-" << std::endl; -#endif - /* call out */ - dhtResultNotify(notifyHash); - - break; - } - - case DHT_MODE_SEARCH: - { - - if (ownhash == key) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() SEARCH msg is OWN PUBLISH -> discarding" << std::endl; -#endif - return false; - } - - uint32_t a1,b1,c1,d1,e1; - uint32_t a2,b2,c2,d2,e2; - uint32_t flags; - - if (sscanf(&((value.c_str())[loc]), " IPL=%d.%d.%d.%d:%d, IPE=%d.%d.%d.%d:%d, type=%x", - &a1, &b1, &c1, &d1, &e1, &a2, &b2, &c2, &d2, &e2, &flags) != 11) - { - /* failed to scan */ -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() SEARCH parse failed of:" << (&((value.c_str())[loc])); - std::cerr << std::endl; -#endif - return false; - } - - std::string out1; - rs_sprintf(out1, "%d.%d.%d.%d", a1, b1, c1, d1); - inet_aton(out1.c_str(), &(laddr.sin_addr)); - laddr.sin_port = htons(e1); - laddr.sin_family = AF_INET; - - std::string out2; - rs_sprintf(out2, "%d.%d.%d.%d", a2, b2, c2, d2); - inet_aton(out2.c_str(), &(raddr.sin_addr)); - raddr.sin_port = htons(e2); - raddr.sin_family = AF_INET; - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() SEARCH laddr: " << out1 << ":" << e1; - std::cerr << " raddr: " << out2 << ":" << e2; - std::cerr << " flags: " << flags; - std::cerr << std::endl; -#endif - - return dhtResultSearch(key, laddr, raddr, flags, sign); - - break; - } - - case DHT_MODE_BOOTSTRAP: - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() BOOTSTRAP msg" << std::endl; -#endif - - /* get the hash */ - std::string bootId = value.substr(loc); -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::resultDHT() BOOTSTRAP msg, IdHash:->" << RsUtil::BinToHex(bootId) << "<-" << std::endl; -#endif - /* call out */ - dhtResultBootstrap(bootId); - - break; - } - - default: - - return false; - break; - } - - return false; -} - - - - -bool p3DhtMgr::dhtResultBootstrap(std::string idhash) -{ - RsStackMutex stack(dhtMtx); /***** LOCK MUTEX *****/ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResultBootstrap() from idhash: "; - std::cerr << RsUtil::BinToHex(idhash) << std::endl; -#endif - - /* Temp - to avoid duplication during testing */ - if (stunIds.end() == std::find(stunIds.begin(), stunIds.end(), idhash)) - { - stunIds.push_back(idhash); -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResultBootstrap() adding to StunList"; - std::cerr << std::endl; -#endif - } - else - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResultBootstrap() DUPLICATE not adding to List"; - std::cerr << std::endl; -#endif - } - - - return true; -} - - - - -bool p3DhtMgr::dhtResultNotify(std::string idhash) -{ - RsStackMutex stack(dhtMtx); /***** LOCK MUTEX *****/ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResultNotify() from idhash: "; - std::cerr << RsUtil::BinToHex(idhash) << std::endl; -#endif - std::map::iterator it; - rstime_t now = time(NULL); - - /* if notify - we must match on the second hash */ - for(it = peers.begin(); (it != peers.end()) && ((it->second).hash2 != idhash); it++) ; - - /* update data */ - /* ignore OFF peers */ - if ((it != peers.end()) && (it->second.state != DHT_PEER_OFF)) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResult() NOTIFY for id: " << it->first << std::endl; -#endif -#ifdef DHT_LOGS - { - /* Log */ - rslog(RSL_WARNING, p3dhtzone, "p3DhtMgr::dhtResultNotify() NOTIFY from Id: " + it->first); - } -#endif - - /* delay callback -> if they are not found */ - it->second.notifyTS = now; - it->second.notifyPending = RS_CONNECT_PASSIVE; - mDhtModifications = true; /* no wait! */ - - if (it->second.state != DHT_PEER_FOUND) - { - /* flag for immediate search */ - it->second.lastTS = 0; - } - } - else - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResult() unknown NOTIFY "; - std::cerr << RsUtil::BinToHex(idhash) << std::endl; -#endif - } - - return true; -} - - -bool p3DhtMgr::dhtResultSearch(std::string idhash, - struct sockaddr_in &laddr, struct sockaddr_in &raddr, - uint32_t type, std::string /*sign*/) -{ - dhtMtx.lock(); /* LOCK MUTEX */ - -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResultSearch() for idhash: "; - std::cerr << RsUtil::BinToHex(idhash) << std::endl; -#endif - std::map::iterator it; - bool doCb = false; - bool doStun = false; - uint32_t stunFlags = 0; - rstime_t now = time(NULL); - - dhtPeerEntry ent; - - /* if search - we must match on the second hash */ - for(it = peers.begin(); (it != peers.end()) && ((it->second).hash1 != idhash); it++) ; - - /* update data */ - /* ignore OFF peers */ - if ((it != peers.end()) && (it->second.state != DHT_PEER_OFF)) - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResult() SEARCH for id: " << it->first << std::endl; -#endif - it->second.lastTS = now; - -#ifdef DHT_LOGS - { - /* Log */ - std::string out ="p3DhtMgr::dhtSearchResult() for Id: " + it->first; - rs_sprintf_append(out, " laddr: %s:%u", rs_inet_ntoa(laddr.sin_addr).c_str(), ntohs(laddr.sin_port)); - rs_sprintf_append(out, " raddr: %s:%u", rs_inet_ntoa(raddr.sin_addr).c_str(), ntohs(raddr.sin_port)); - rs_sprintf_append(out, " type: %lu", ownEntry.type); - - rslog(RSL_WARNING, p3dhtzone, out); - } -#endif - - /* update info .... always */ - it->second.state = DHT_PEER_FOUND; - it->second.laddr = laddr; - it->second.raddr = raddr; - it->second.type = type; - - /* Do callback all the time */ - ent = it->second; - doCb = true; - - if (it->second.notifyPending) - { - /* no wait if we have pendingNotification */ - mDhtModifications = true; - } - - /* do Stun always */ - doStun = true; - stunFlags = RS_STUN_FRIEND | RS_STUN_ONLINE; - } - else - { -#ifdef DHT_DEBUG - std::cerr << "p3DhtMgr::dhtResult() SEARCH(stun) for idhash: "; - std::cerr << RsUtil::BinToHex(idhash) << std::endl; -#endif - /* stun result? */ - doStun = true; - stunFlags = RS_STUN_ONLINE; - } - - dhtMtx.unlock(); /* UNLOCK MUTEX */ - - if (doCb) - { - pqiIpAddrSet addrs; - - pqiIpAddress laddr; - laddr.mAddr = ent.laddr; - laddr.mSeenTime = time(NULL); - laddr.mSrc = RS_CB_DHT; - - addrs.updateLocalAddrs(laddr); - - pqiIpAddress eaddr; - eaddr.mAddr = ent.raddr; - eaddr.mSeenTime = time(NULL); - eaddr.mSrc = RS_CB_DHT; - - addrs.updateExtAddrs(eaddr); - - mConnCb->peerStatus(ent.id, addrs, ent.type, 0, RS_CB_DHT); - } - - if (doStun) - { - //mConnCb->stunStatus(idhash, raddr, type, stunFlags); - } - - return true; -} - - - -/******************************** AUX FUNCTIONS ********************************** - * - */ - -void printDhtPeerEntry(dhtPeerEntry *ent, std::ostream &out) -{ - - out << "DhtEntry: ID: " << ent->id.toStdString(); - out << " State: " << ent->state; - out << " lastTS: " << ent->lastTS; - out << " notifyPending: " << ent->notifyPending; - out << " notifyTS: " << ent->notifyTS; - out << std::endl; - out << " laddr: " << rs_inet_ntoa(ent->laddr.sin_addr) << ":" << ntohs(ent->laddr.sin_port); - out << " raddr: " << rs_inet_ntoa(ent->raddr.sin_addr) << ":" << ntohs(ent->raddr.sin_port); - out << " type: " << ent->type; - out << std::endl; - out << " hash1: " << RsUtil::BinToHex(ent->hash1); - out << std::endl; - out << " hash2: " << RsUtil::BinToHex(ent->hash2); - out << std::endl; - return; -} - - diff --git a/libretroshare/src/pqi/p3dhtmgr.h b/libretroshare/src/pqi/p3dhtmgr.h deleted file mode 100644 index b0b2fb7b1..000000000 --- a/libretroshare/src/pqi/p3dhtmgr.h +++ /dev/null @@ -1,246 +0,0 @@ -/******************************************************************************* - * libretroshare/src/pqi: p3dhtmgr.h * - * * - * libretroshare: retroshare core library * - * * - * Copyright 2004-2008 by Robert Fernie. * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public License * - * along with this program. If not, see . * - * * - *******************************************************************************/ - -#ifndef MRK_P3_DHT_MANAGER_HEADER -#define MRK_P3_DHT_MANAGER_HEADER - -/* Interface class for DHT data */ - -#include -#include -#include "pqi/pqinetwork.h" - -#include "util/rsthreads.h" -#include "pqi/pqimonitor.h" - -#include "pqi/pqiassist.h" - -/* All other #defs are in .cc */ -#define DHT_ADDR_INVALID 0xff -#define DHT_ADDR_TCP 0x01 -#define DHT_ADDR_UDP 0x02 - - -/* for DHT peer STATE */ -#define DHT_PEER_OFF 0 -#define DHT_PEER_INIT 1 -#define DHT_PEER_SEARCH 2 -#define DHT_PEER_FOUND 3 - -/* for DHT peer STATE (ownEntry) */ -#define DHT_PEER_ADDR_KNOWN 4 -#define DHT_PEER_PUBLISHED 5 - -/* Interface with Real DHT Implementation */ -#define DHT_MODE_SEARCH 1 -#define DHT_MODE_PUBLISH 1 -#define DHT_MODE_NOTIFY 2 -#define DHT_MODE_BOOTSTRAP 3 - - -/* TIMEOUTS: Reference Values are set here... */ - -#define DHT_SEARCH_PERIOD 1800 /* PeerKeys: if we haven't found them: 30 min */ -#define DHT_CHECK_PERIOD 1800 /* PeerKeys: re-lookup peer: 30 min */ -#define DHT_PUBLISH_PERIOD 1800 /* OwnKey: 30 min */ -#define DHT_NOTIFY_PERIOD 300 /* 5 min - Notify Check period */ - -/* TTLs for DHTs posts */ -#define DHT_TTL_PUBLISH (DHT_PUBLISH_PERIOD + 120) // for a little overlap. -#define DHT_TTL_NOTIFY (DHT_NOTIFY_PERIOD + 60) // for time to find it... -#define DHT_TTL_BOOTSTRAP (DHT_PUBLISH_PERIOD) // To start with. - -class dhtPeerEntry -{ - public: - dhtPeerEntry(); - - RsPeerId id; - uint32_t state; - rstime_t lastTS; - - uint32_t notifyPending; - rstime_t notifyTS; - - struct sockaddr_in laddr, raddr; - uint32_t type; /* ADDR_TYPE as defined above */ - - std::string hash1; /* SHA1 Hash of id */ - std::string hash2; /* SHA1 Hash of reverse Id */ -}; - -class p3DhtMgr: public pqiNetAssistConnect, public RsThread -{ - /* - */ - public: - p3DhtMgr(RsPeerId id, pqiConnectCb *cb); - - /********** External DHT Interface ************************ - * These Functions are the external interface - * for the DHT, and must be non-blocking and return quickly - */ - - /* OVERLOADED From pqiNetAssistConnect. */ - -virtual void enable(bool on); -virtual void shutdown(); -virtual void restart(); - -virtual bool getEnabled(); /* on */ -virtual bool getActive(); /* actually working */ - -virtual void setBootstrapAllowed(bool on); -virtual bool getBootstrapAllowed(); - - /* set key data */ -virtual bool setExternalInterface(struct sockaddr_in laddr, - struct sockaddr_in raddr, uint32_t type); - - /* add / remove peers */ -virtual bool findPeer(const RsPeerId& id); -virtual bool dropPeer(const RsPeerId& id); - - /* post DHT key saying we should connect (callback when done) */ -virtual bool notifyPeer(const RsPeerId& id); - - /* extract current peer status */ -virtual bool getPeerStatus(const RsPeerId& id, - struct sockaddr_storage &laddr, struct sockaddr_storage &raddr, - uint32_t &type, uint32_t &mode); - - /* stun */ -virtual bool enableStun(bool on); -virtual bool addStun(std::string id); - //doneStun(); - - /********** Higher Level DHT Work Functions ************************ - * These functions translate from the strings/addresss to - * key/value pairs. - */ - public: - - /* results from DHT proper */ -virtual bool dhtResultNotify(std::string id); -virtual bool dhtResultSearch(std::string id, - struct sockaddr_in &laddr, struct sockaddr_in &raddr, - uint32_t type, std::string sign); - -virtual bool dhtResultBootstrap(std::string idhash); - - protected: - - /* can block briefly (called only from thread) */ -virtual bool dhtPublish(std::string idhash, - struct sockaddr_in &laddr, - struct sockaddr_in &raddr, - uint32_t type, std::string sign); - -virtual bool dhtNotify(std::string idhash, std::string ownIdHash, - std::string sign); - -virtual bool dhtSearch(std::string idhash, uint32_t mode); - -virtual bool dhtBootstrap(std::string idhash, std::string ownIdHash, - std::string sign); /* to publish bootstrap */ - - - - /********** Actual DHT Work Functions ************************ - * These involve a very simple LOW-LEVEL interface ... - * - * publish - * search - * result - * - */ - - public: - - /* Feedback callback (handled here) */ -virtual bool resultDHT(std::string key, std::string value); - - protected: - -virtual bool dhtInit(); -virtual bool dhtShutdown(); -virtual bool dhtActive(); -virtual int status(std::ostream &out); - -virtual bool publishDHT(std::string key, std::string value, uint32_t ttl); -virtual bool searchDHT(std::string key); - - - - /********** Internal DHT Threading ************************ - * - */ - - public: - -virtual void run(); - - private: - - /* search scheduling */ -void checkDHTStatus(); -int checkStunState(); -int checkStunState_Active(); /* when in active state */ -int doStun(); -int checkPeerDHTKeys(); -int checkOwnDHTKeys(); -int checkNotifyDHT(); - -void clearDhtData(); - - /* IP Bootstrap */ -bool getDhtBootstrapList(); -std::string BootstrapId(uint32_t bin); -std::string randomBootstrapId(); - - /* other feedback through callback */ - // use pqiNetAssistConnect.. version pqiConnectCb *connCb; - - /* protected by Mutex */ - RsMutex dhtMtx; - - bool mDhtOn; /* User desired state */ - bool mDhtModifications; /* any user requests? */ - - dhtPeerEntry ownEntry; - rstime_t ownNotifyTS; - std::map peers; - - std::list stunIds; - bool mStunRequired; - - uint32_t mDhtState; - rstime_t mDhtActiveTS; - - bool mBootstrapAllowed; - rstime_t mLastBootstrapListTS; -}; - - -#endif // MRK_P3_DHT_MANAGER_HEADER - - diff --git a/libretroshare/src/pqi/p3linkmgr.cc b/libretroshare/src/pqi/p3linkmgr.cc index fec5b8e5f..100a16cad 100644 --- a/libretroshare/src/pqi/p3linkmgr.cc +++ b/libretroshare/src/pqi/p3linkmgr.cc @@ -28,7 +28,6 @@ #include "rsserver/p3face.h" #include "pqi/authssl.h" -#include "pqi/p3dhtmgr.h" // Only need it for constants. #include "tcponudp/tou.h" #include "util/extaddrfinder.h" #include "util/dnsresolver.h" diff --git a/libretroshare/src/pqi/p3netmgr.cc b/libretroshare/src/pqi/p3netmgr.cc index 4a539701c..8eebd4c1e 100644 --- a/libretroshare/src/pqi/p3netmgr.cc +++ b/libretroshare/src/pqi/p3netmgr.cc @@ -79,16 +79,12 @@ const uint32_t MAX_UPNP_COMPLETE = 600; /* 10 min... seems to take a while */ // #define NETMGR_DEBUG_TICK 1 // #define NETMGR_DEBUG_STATEBOX 1 -pqiNetStatus::pqiNetStatus() - :mLocalAddrOk(false), mExtAddrOk(false), mExtAddrStableOk(false), - mUpnpOk(false), mDhtOk(false), mResetReq(false) +pqiNetStatus::pqiNetStatus() : + mExtAddrOk(false), mExtAddrStableOk(false), mUpnpOk(false), mDhtOk(false), + mDhtNetworkSize(0), mDhtRsNetworkSize(0), mResetReq(false) { - mDhtNetworkSize = 0; - mDhtRsNetworkSize = 0; - sockaddr_storage_clear(mLocalAddr); sockaddr_storage_clear(mExtAddr); - return; } @@ -96,7 +92,6 @@ pqiNetStatus::pqiNetStatus() void pqiNetStatus::print(std::ostream &out) { out << "pqiNetStatus: "; - out << "mLocalAddrOk: " << mLocalAddrOk; out << " mExtAddrOk: " << mExtAddrOk; out << " mExtAddrStableOk: " << mExtAddrStableOk; out << std::endl; @@ -108,14 +103,13 @@ void pqiNetStatus::print(std::ostream &out) out << std::endl; out << "mLocalAddr: " << sockaddr_storage_tostring(mLocalAddr) << " "; out << "mExtAddr: " << sockaddr_storage_tostring(mExtAddr) << " "; - out << " NetOk: " << NetOk(); out << std::endl; } -p3NetMgrIMPL::p3NetMgrIMPL() - :mPeerMgr(NULL), mLinkMgr(NULL), mNetMtx("p3NetMgr"), - mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false) +p3NetMgrIMPL::p3NetMgrIMPL() : mPeerMgr(nullptr), mLinkMgr(nullptr), + mNetMtx("p3NetMgr"), mNetStatus(RS_NET_UNKNOWN), mStatusChanged(false), + mDoNotNetCheckUntilTs(0) { { @@ -130,7 +124,6 @@ p3NetMgrIMPL::p3NetMgrIMPL() mNetFlags = pqiNetStatus(); mOldNetFlags = pqiNetStatus(); - mLastSlowTickTime = 0; mOldNatType = RSNET_NATTYPE_UNKNOWN; mOldNatHole = RSNET_NATHOLE_UNKNOWN; sockaddr_storage_clear(mLocalAddr); @@ -415,47 +408,37 @@ void p3NetMgrIMPL::netStartup() void p3NetMgrIMPL::tick() { - rstime_t now = time(NULL); - bool doSlowTick = false; + rstime_t now = time(nullptr); + rstime_t dontCheckNetUntil; + { RS_STACK_MUTEX(mNetMtx); dontCheckNetUntil = mDoNotNetCheckUntilTs; } + + if(now >= dontCheckNetUntil) netStatusTick(); + + uint32_t netStatus; { RS_STACK_MUTEX(mNetMtx); netStatus = mNetStatus; } + switch (netStatus) { - RsStackMutex stack(mNetMtx); /************** LOCK MUTEX ***************/ - if (now > mLastSlowTickTime) + case RS_NET_LOOPBACK: + if(dontCheckNetUntil <= now) { - mLastSlowTickTime = now; - doSlowTick = true; + RS_STACK_MUTEX(mNetMtx); + mDoNotNetCheckUntilTs = now + 30; } - } - - if (doSlowTick) - { - slowTick(); - } -} - - -void p3NetMgrIMPL::slowTick() -{ - netTick(); - netAssistTick(); - updateNetStateBox_temporal(); - + break; + default: + netAssistTick(); + updateNetStateBox_temporal(); #ifdef RS_USE_DHT_STUNNER - if (mDhtStunner) - { - mDhtStunner->tick(); - } - - if (mProxyStunner) - { - mProxyStunner->tick(); - } + if (mDhtStunner) mDhtStunner->tick(); + if (mProxyStunner) mProxyStunner->tick(); #endif // RS_USE_DHT_STUNNER + break; + } } #define STARTUP_DELAY 5 -void p3NetMgrIMPL::netTick() +void p3NetMgrIMPL::netStatusTick() { #ifdef NETMGR_DEBUG_TICK @@ -986,10 +969,9 @@ bool p3NetMgrIMPL::checkNetAddress() if (mNetMode & RS_NET_MODE_TRY_LOOPBACK) { -#if defined(NETMGR_DEBUG_TICK) || defined(NETMGR_DEBUG_RESET) - std::cerr << "p3NetMgrIMPL::checkNetAddress() LOOPBACK ... forcing to 127.0.0.1"; - std::cerr << std::endl; -#endif + RsInfo() << __PRETTY_FUNCTION__ <<" network mode set to LOOPBACK," + << " forcing address to 127.0.0.1" << std::endl; + sockaddr_storage_ipv4_aton(prefAddr, "127.0.0.1"); validAddr = true; } @@ -1034,137 +1016,77 @@ bool p3NetMgrIMPL::checkNetAddress() break; } } + + /* If no satisfactory local address has been found yet relax and + * accept also loopback addresses */ + if(!validAddr) for (auto it = addrs.begin(); it!=addrs.end(); ++it) + { + sockaddr_storage& addr(*it); + if( sockaddr_storage_isValidNet(addr) && + sockaddr_storage_ipv6_to_ipv4(addr) ) + { + prefAddr = addr; + validAddr = true; + break; + } + } } } if (!validAddr) { - Dbg3() << __PRETTY_FUNCTION__ << " no valid local network address found" - << std::endl; + RsErr() << __PRETTY_FUNCTION__ << " no valid local network address " + <<" found. Report to developers." << std::endl; + print_stacktrace(); + return false; } - /* check addresses */ - - { - RS_STACK_MUTEX(mNetMtx); + { RS_STACK_MUTEX(mNetMtx); sockaddr_storage_copy(mLocalAddr, oldAddr); addrChanged = !sockaddr_storage_sameip(prefAddr, mLocalAddr); -#ifdef NETMGR_DEBUG_TICK - std::cerr << "p3NetMgrIMPL::checkNetAddress()"; - std::cerr << std::endl; - std::cerr << "Current Local: " << sockaddr_storage_tostring(mLocalAddr); - std::cerr << std::endl; - std::cerr << "Current Preferred: " << sockaddr_storage_iptostring(prefAddr); - std::cerr << std::endl; -#endif - -#ifdef NETMGR_DEBUG_RESET - if (addrChanged) - { - std::cerr << "p3NetMgrIMPL::checkNetAddress() Address Changed!"; - std::cerr << std::endl; - std::cerr << "Current Local: " << sockaddr_storage_tostring(mLocalAddr); - std::cerr << std::endl; - std::cerr << "Current Preferred: " << sockaddr_storage_iptostring(prefAddr); - std::cerr << std::endl; - } -#endif - // update address. sockaddr_storage_copyip(mLocalAddr, prefAddr); sockaddr_storage_copy(mLocalAddr, mNetFlags.mLocalAddr); if(sockaddr_storage_isLoopbackNet(mLocalAddr)) - { -#ifdef NETMGR_DEBUG - std::cerr << "p3NetMgrIMPL::checkNetAddress() laddr: Loopback" << std::endl; -#endif - mNetFlags.mLocalAddrOk = false; mNetStatus = RS_NET_LOOPBACK; - } - else if (!sockaddr_storage_isValidNet(mLocalAddr)) - { -#ifdef NETMGR_DEBUG - std::cerr << "p3NetMgrIMPL::checkNetAddress() laddr: invalid" << std::endl; -#endif - mNetFlags.mLocalAddrOk = false; - } - else - { -#ifdef NETMGR_DEBUG_TICK - std::cerr << "p3NetMgrIMPL::checkNetAddress() laddr okay" << std::endl; -#endif - mNetFlags.mLocalAddrOk = true; - } - - int port = sockaddr_storage_port(mLocalAddr); - if ((port < PQI_MIN_PORT) || (port > PQI_MAX_PORT)) + // Check if local port is valid, reset it if not + if (!sockaddr_storage_port(mLocalAddr)) { -#ifdef NETMGR_DEBUG - std::cerr << "p3NetMgrIMPL::checkNetAddress() Correcting Port to DEFAULT" << std::endl; -#endif - uint16_t new_port = htons(PQI_MIN_PORT_RNG + (RSRandom::random_u32() % (PQI_MAX_PORT - PQI_MIN_PORT_RNG))); - sockaddr_storage_setport(mLocalAddr, new_port); + /* Using same port as external may make some NAT happier */ + uint16_t port = sockaddr_storage_port(mExtAddr); + /* Avoid to automatically set a local port to a reserved one < 1024 + * that needs special permissions or root access. + * This do not impede the user to set a reserved port manually, + * which make sense in some cases. */ + while (port < 1025) + port = static_cast(RsRandom::random_u32()); + + sockaddr_storage_setport(mLocalAddr, htons(port)); addrChanged = true; + + RsWarn() << __PRETTY_FUNCTION__ << " local port was 0, corrected " + <<"to: " << port << std::endl; } - -#if DEAD_CODE - /* Enabling this piece of code breaks setup where an additional BOFH - * overlooked port like 80 or 443 is manually forwarded to RetroShare to - * avoid restrictive firewals. - * In the case of a real mismatch, it is not really problematic, as our - * peers would get and then attempt to connect also to the right port. - */ - - /* if localaddr == serveraddr, then ensure that the ports - * are the same (modify server)... this mismatch can - * occur when the local port is changed.... - */ - if ( sockaddr_storage_sameip(mLocalAddr, mExtAddr) - && sockaddr_storage_port(mLocalAddr) != sockaddr_storage_port(mExtAddr) ) - { -#ifdef NETMGR_DEBUG_RESET - std::cerr << __PRETTY_FUNCTION__ << " local and external ports are" - << " not the same. Setting external port to " - << sockaddr_storage_port(mLocalAddr) << std::endl; -#endif - sockaddr_storage_setport(mExtAddr, sockaddr_storage_port(mLocalAddr)); - addrChanged = true; - } -#endif // DEAD_CODE - -#ifdef NETMGR_DEBUG_TICK - std::cerr << __PRETTY_FUNCTION__ << " Final Local Address: " - << sockaddr_storage_tostring(mLocalAddr) << std::endl; -#endif - - } + } // RS_STACK_MUTEX(mNetMtx); if (addrChanged) { -#ifdef NETMGR_DEBUG_RESET - std::cerr << "p3NetMgrIMPL::checkNetAddress() local address changed, resetting network." << std::endl; - std::cerr << std::endl; -#endif + RsInfo() << __PRETTY_FUNCTION__ << " local address changed, resetting" + <<" network." << std::endl; - if (mPeerMgr) - { - mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr); - } - - std::cerr << __PRETTY_FUNCTION__ - << " local address changed, resetting network" << std::endl; + if (mPeerMgr) mPeerMgr->UpdateOwnAddress(mLocalAddr, mExtAddr); netReset(); } - return 1; + return true; } @@ -2051,3 +1973,6 @@ void p3NetMgrIMPL::updateNetStateBox_reset() } p3NetMgr::~p3NetMgr() = default; +pqiNetAssist::~pqiNetAssist() = default; +pqiNetAssistPeerShare::~pqiNetAssistPeerShare() = default; +pqiNetAssistConnect::~pqiNetAssistConnect() = default; diff --git a/libretroshare/src/pqi/p3netmgr.h b/libretroshare/src/pqi/p3netmgr.h index cf2705d60..f7775757d 100644 --- a/libretroshare/src/pqi/p3netmgr.h +++ b/libretroshare/src/pqi/p3netmgr.h @@ -44,13 +44,10 @@ class DNSResolver ; -class pqiNetStatus +struct pqiNetStatus { - public: - pqiNetStatus(); - bool mLocalAddrOk; // Local address is not loopback. bool mExtAddrOk; // have external address. bool mExtAddrStableOk; // stable external address. bool mUpnpOk; // upnp is ok. @@ -65,11 +62,6 @@ class pqiNetStatus bool mResetReq; // Not Used yet!. void print(std::ostream &out); - - bool NetOk() // minimum to believe network is okay.` - { - return (mLocalAddrOk && mExtAddrOk); - } }; class p3PeerMgr; @@ -211,11 +203,6 @@ void addNetListener(pqiNetListener *listener); // SHOULD MAKE THIS PROTECTED. bool checkNetAddress(); /* check our address is sensible */ -protected: - -void slowTick(); - - protected: /****************** Internal Interface *******************/ bool enableNetAssistFirewall(bool on); @@ -248,7 +235,7 @@ bool netAssistAttach(bool on); void netReset(); void statusTick(); -void netTick(); +void netStatusTick(); void netStartup(); /* startup the bits */ @@ -335,7 +322,7 @@ void netStatusReset_locked(); // Improved NetStatusBox, which uses the Stunners! pqiNetStateBox mNetStateBox; - rstime_t mLastSlowTickTime; + rstime_t mDoNotNetCheckUntilTs; uint32_t mOldNatType; uint32_t mOldNatHole; diff --git a/libretroshare/src/pqi/p3peermgr.cc b/libretroshare/src/pqi/p3peermgr.cc index e871e4ddd..564eb44b8 100644 --- a/libretroshare/src/pqi/p3peermgr.cc +++ b/libretroshare/src/pqi/p3peermgr.cc @@ -3,8 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright (C) 2007-2011 Robert Fernie * - * Copyright (C) 2015-2018 Gioacchino Mazzurco * + * Copyright (C) 2007-2011 Robert Fernie * + * Copyright (C) 2015-2019 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -34,11 +34,6 @@ #include "pqi/p3historymgr.h" #include "pqi/pqinetwork.h" // for getLocalAddresses -//#include "pqi/p3dhtmgr.h" // Only need it for constants. -//#include "tcponudp/tou.h" -//#include "util/extaddrfinder.h" -//#include "util/dnsresolver.h" - #include "util/rsprint.h" #include "util/rsstring.h" #include "util/rsdebug.h" @@ -1175,8 +1170,8 @@ bool p3PeerMgrIMPL::addSslOnlyFriend( * previously known IP addresses */ if(!dt.isHiddenNode) { - for(const std::string& locator : dt.ipAddressList) - addPeerLocator(sslId, locator); + for(const std::string& ipStr : dt.ipAddressList) + addPeerLocator(sslId, RsUrl(ipStr)); if(dt.extPort && !dt.extAddr.empty()) { @@ -3159,4 +3154,4 @@ bool p3PeerMgrIMPL::removeUnusedLocations() return true; } - +p3PeerMgr::~p3PeerMgr() = default; diff --git a/libretroshare/src/pqi/p3peermgr.h b/libretroshare/src/pqi/p3peermgr.h index d5451eaa9..2cc9e4888 100644 --- a/libretroshare/src/pqi/p3peermgr.h +++ b/libretroshare/src/pqi/p3peermgr.h @@ -3,7 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2007-2011 by Robert Fernie * + * Copyright (C) 2007-2011 Robert Fernie * + * Copyright (C) 2015-2019 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -112,8 +113,6 @@ class peerState class RsNodeGroupItem; struct RsGroupInfo; -std::string textPeerState(peerState &state); - class p3LinkMgr; class p3NetMgr; @@ -123,10 +122,6 @@ class p3NetMgrIMPL; class p3PeerMgr { public: - - p3PeerMgr() {} - virtual ~p3PeerMgr() {} - virtual bool addFriend( const RsPeerId &ssl_id, const RsPgpId &gpg_id, uint32_t netMode = RS_NET_MODE_UDP, uint16_t vsDisc = RS_VS_DISC_FULL, @@ -242,7 +237,7 @@ virtual bool locked_computeCurrentBestOwnExtAddressCandidate(sockaddr_storage /*************************************************************************************************/ /*************************************************************************************************/ - + virtual ~p3PeerMgr(); }; diff --git a/libretroshare/src/pqi/pqi_base.h b/libretroshare/src/pqi/pqi_base.h index e220a376d..51980b501 100644 --- a/libretroshare/src/pqi/pqi_base.h +++ b/libretroshare/src/pqi/pqi_base.h @@ -37,12 +37,6 @@ struct RSTrafficClue; #include "serialiser/rsserial.h" #include "retroshare/rstypes.h" - -#define PQI_MIN_PORT 10 // TO ALLOW USERS TO HAVE PORT 80! - was 1024 -#define PQI_MIN_PORT_RNG 1024 -#define PQI_MAX_PORT 65535 -#define PQI_DEFAULT_PORT 7812 - int getPQIsearchId(); int fixme(char *str, int n); diff --git a/libretroshare/src/pqi/pqiassist.h b/libretroshare/src/pqi/pqiassist.h index 1bc00fbe9..e7b508a3e 100644 --- a/libretroshare/src/pqi/pqiassist.h +++ b/libretroshare/src/pqi/pqiassist.h @@ -19,8 +19,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef MRK_PQI_ASSIST_H -#define MRK_PQI_ASSIST_H +#pragma once #include #include @@ -28,25 +27,15 @@ #include "pqi/pqinetwork.h" #include "pqi/pqimonitor.h" -/********** - * This header file provides two interfaces for assisting - * the connections to friends. - * - * pqiNetAssistFirewall - which will provides interfaces - * to functionality like upnp and apple's equivalent. - * - * pqiNetAssistConnect - which will provides interfaces - * to other networks (DHT) etc that can provide information. - * These classes would be expected to use the pqiMonitor - * callback system to notify the connectionMgr. - * - ***/ +/** + * @file + * This header file provides interfaces for assisting the connections to + * friends. + */ class pqiNetAssist { - public: - -virtual ~pqiNetAssist() { return; } +public: /* External Interface */ virtual void enable(bool on) = 0; @@ -58,28 +47,24 @@ virtual bool getActive() = 0; virtual int tick() { return 0; } /* for internal accounting */ + virtual ~pqiNetAssist(); }; - -#define PFP_TYPE_UDP 0x0001 -#define PFP_TYPE_TCP 0x0002 - -class PortForwardParams +struct PortForwardParams { - public: uint32_t fwdId; uint32_t status; uint32_t typeFlags; - struct sockaddr_storage intAddr; - struct sockaddr_storage extaddr; + sockaddr_storage intAddr; + sockaddr_storage extaddr; }; +/** + * Provides interfaces to functionality like upnp and apple's equivalent. + */ class pqiNetAssistFirewall: public pqiNetAssist { - public: - -virtual ~pqiNetAssistFirewall() { return; } - +public: /* the address that the listening port is on */ virtual void setInternalPort(unsigned short iport_in) = 0; virtual void setExternalPort(unsigned short eport_in) = 0; @@ -102,11 +87,13 @@ virtual bool statusPortForward(const uint32_t fwdId, PortForwardParams ¶m class pqiNetAssistPeerShare { - public: - - /* share Addresses for various reasons (bad peers, etc) */ -virtual void updatePeer(const RsPeerId& id, const struct sockaddr_storage &addr, int type, int reason, int age) = 0; +public: + /** share Addresses for various reasons (bad peers, etc) */ + virtual void updatePeer( + const RsPeerId& id, const struct sockaddr_storage &addr, + int type, int reason, int age ) = 0; + virtual ~pqiNetAssistPeerShare(); }; @@ -141,20 +128,22 @@ virtual int tick() = 0; /* for internal accounting */ #define NETASSIST_KNOWN_PEER_TYPE_MASK 0xff00 +/** + * Provides interfaces to other networks like DHT that can provide information. + * These classes would be expected to use the pqiMonitor callback system to + * notify the connectionMgr. + */ class pqiNetAssistConnect: public pqiNetAssist { - /* - */ - public: - pqiNetAssistConnect(const RsPeerId& id, pqiConnectCb *cb) - :mPeerId(id), mConnCb(cb) { return; } +public: + pqiNetAssistConnect(const RsPeerId& id, pqiConnectCb *cb) : + mPeerId(id), mConnCb(cb) {} /********** External DHT Interface ************************ * These Functions are the external interface * for the DHT, and must be non-blocking and return quickly */ - /* add / remove peers */ virtual bool findPeer(const RsPeerId& id) = 0; virtual bool dropPeer(const RsPeerId& id) = 0; @@ -175,11 +164,9 @@ virtual bool setAttachMode(bool on) = 0; // FIXUP. /***** Stats for Network / DHT *****/ virtual bool getNetworkStats(uint32_t &netsize, uint32_t &localnetsize) = 0; // DEPRECIATE. - protected: + ~pqiNetAssistConnect() override; + +protected: RsPeerId mPeerId; pqiConnectCb *mConnCb; }; - - -#endif /* MRK_PQI_ASSIST_H */ - diff --git a/libretroshare/src/pqi/pqimonitor.h b/libretroshare/src/pqi/pqimonitor.h index eaba16b73..34a858e66 100644 --- a/libretroshare/src/pqi/pqimonitor.h +++ b/libretroshare/src/pqi/pqimonitor.h @@ -63,10 +63,6 @@ const uint32_t RS_STUN_FRIEND = 0x0020; const uint32_t RS_STUN_FRIEND_OF_FRIEND = 0x0040; -// for the old p3dhtmgr - amazed it still compiles ;) -#define RS_CONNECT_PASSIVE 1 -#define RS_CONNECT_ACTIVE 2 - #define RS_CB_DHT 0x0001 /* from dht */ #define RS_CB_DISC 0x0002 /* from peers */ #define RS_CB_PERSON 0x0003 /* from connection */ diff --git a/libretroshare/src/pqi/pqissl.h b/libretroshare/src/pqi/pqissl.h index 6246145b7..3f315e329 100644 --- a/libretroshare/src/pqi/pqissl.h +++ b/libretroshare/src/pqi/pqissl.h @@ -75,9 +75,6 @@ const int WINDOWS_TCP_BUFFER_SIZE = 512 * 1024; // 512 KB * */ -class pqissl; -class cert; - class pqissllistener; class p3LinkMgr; struct RsPeerCryptoParams; diff --git a/libretroshare/src/pqi/pqisslproxy.h b/libretroshare/src/pqi/pqisslproxy.h index d643ccb2e..dc34295ad 100644 --- a/libretroshare/src/pqi/pqisslproxy.h +++ b/libretroshare/src/pqi/pqisslproxy.h @@ -34,9 +34,6 @@ * It uses the Domain Name interface of SOCKS5, as opposed to an IP address. */ -class pqisslproxy; -class cert; - /* This provides a NetBinInterface, which is * primarily inherited from pqissl. * fns declared here are different -> all others are identical. diff --git a/libretroshare/src/retroshare/rsbroadcastdiscovery.h b/libretroshare/src/retroshare/rsbroadcastdiscovery.h index a712c60af..1bb1e1cc6 100644 --- a/libretroshare/src/retroshare/rsbroadcastdiscovery.h +++ b/libretroshare/src/retroshare/rsbroadcastdiscovery.h @@ -107,5 +107,45 @@ public: */ virtual std::vector getDiscoveredPeers() = 0; + /** + * @brief Check if multicast listening is enabled + * @jsonapi{development} + * On some platforms such as Android multicast listening, which is needed + * for broadcast discovery, is not enabled by default at WiFi driver level + * @see enableMulticastListening, so this method check if it is enabled. + * On platforms that are not expected to have such a limitation this method + * always return true. + * @return true if enabled, false otherwise. + */ + virtual bool isMulticastListeningEnabled() = 0; + + /** + * @brief On platforms that need it enable low level multicast listening + * @jsonapi{development} + * On Android and potencially other mobile platforms, WiFi drivers are + * configured by default to discard any packet that is not directed to the + * unicast mac address of the interface, this way they could save some + * battery but breaks anything that is not unicast, such as broadcast + * discovery. On such platforms this method enable low level multicast + * listening so we can receive advertisement from same broadcast domain + * nodes. + * On platforms without such limitation does nothing and always return + * false. + * It is exposed as a public API so the UI can decide the right equilibrium + * between discoverability and battery saving. + * @return true if multicast listening has been enabled due to this call, + * false otherwise. + */ + virtual bool enableMulticastListening() = 0; + + /** + * @brief Disable multicast listening + * @jsonapi{development} + * The opposite of @see enableMulticastListening. + * @return true if multicast listening has been disabled due to this call, + * false otherwise. + */ + virtual bool disableMulticastListening() = 0; + virtual ~RsBroadcastDiscovery(); }; diff --git a/libretroshare/src/retroshare/rsconfig.h b/libretroshare/src/retroshare/rsconfig.h index 88177df2d..b1522330a 100644 --- a/libretroshare/src/retroshare/rsconfig.h +++ b/libretroshare/src/retroshare/rsconfig.h @@ -206,11 +206,11 @@ struct RSTrafficClue : RsSerializable struct RsConfigNetStatus : RsSerializable { - RsConfigNetStatus() + RsConfigNetStatus() : netLocalOk(true) { localPort = extPort = 0 ; firewalled = forwardPort = false ; - DHTActive = uPnPActive = netLocalOk = netUpnpOk = netDhtOk = netStunOk = netExtAddressOk = false ; + DHTActive = uPnPActive = netUpnpOk = netDhtOk = netStunOk = netExtAddressOk = false ; uPnPState = 0 ; //DHTPeers = 0 ; netDhtNetSize = netDhtRsNetSize = 0; @@ -235,7 +235,8 @@ struct RsConfigNetStatus : RsSerializable int uPnPState; /* Flags for Network Status */ - bool netLocalOk; /* That we've talked to someone! */ + RS_DEPRECATED + bool netLocalOk; /// As of today it's meaningless bool netUpnpOk; /* upnp is enabled and active */ bool netDhtOk; /* response from dht */ bool netStunOk; /* recvd stun / udp packets */ diff --git a/libretroshare/src/retroshare/rsevents.h b/libretroshare/src/retroshare/rsevents.h index 880cade56..dbb4a4cee 100644 --- a/libretroshare/src/retroshare/rsevents.h +++ b/libretroshare/src/retroshare/rsevents.h @@ -66,6 +66,9 @@ enum class RsEventType : uint32_t /// Emitted when a peer state changes, @see RsPeers PEER_STATE_CHANGED = 6, + /// @see RsMailStatusEvent + MAIL_STATUS_CHANGE = 7, + MAX /// Used to detect invalid event type passed }; diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index 47c464052..daa83d3bc 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -202,6 +202,52 @@ struct BannedFileEntry : RsSerializable } }; +struct DeepFilesSearchResult; + +struct TurtleFileInfoV2 : RsSerializable +{ + TurtleFileInfoV2() : fSize(0), fWeight(0) {} + + TurtleFileInfoV2(const TurtleFileInfo& oldInfo) : + fSize(oldInfo.size), fHash(oldInfo.hash), fName(oldInfo.name), + fWeight(0) {} + +#ifdef RS_DEEP_FILES_INDEX + TurtleFileInfoV2(const DeepFilesSearchResult& dRes); +#endif // def RS_DEEP_FILES_INDEX + + uint64_t fSize; /// File size + RsFileHash fHash; /// File hash + std::string fName; /// File name + + /** @brief Xapian weight of the file which matched the search criteria + * This field is optional (its value is 0 when not specified). + * Given that Xapian weight for the same file is usually different on + * different nodes, it should not be used as an absolute refence, but just + * as an hint of how much the given file match the search criteria. + */ + float fWeight; + + /** @brief Xapian snippet of the file which matched the search criteria + * This field is optional (its value is an empty string when not specified). + */ + std::string fSnippet; + + + /// @see RsSerializable::serial_process + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(fSize); + RS_SERIAL_PROCESS(fHash); + RS_SERIAL_PROCESS(fName); + RS_SERIAL_PROCESS(fWeight); + RS_SERIAL_PROCESS(fSnippet); + } + + ~TurtleFileInfoV2() override; +}; + class RsFiles { public: @@ -209,7 +255,7 @@ public: virtual ~RsFiles() {} /** - * @brief Provides file data for the gui, media streaming or rpc clients. + * @brief Provides file data for the GUI, media streaming or API clients. * It may return unverified chunks. This allows streaming without having to * wait for hashes or completion of the file. * This function returns an unspecified amount of bytes. Either as much data @@ -217,8 +263,8 @@ public: * To get more data, call this function repeatedly with different offsets. * * @jsonapi{development,manualwrapper} - * note the missing @ the wrapper for this is written manually not - * autogenerated @see JsonApiServer. + * note the wrapper for this is written manually not autogenerated + * @see JsonApiServer. * * @param[in] hash hash of the file. The file has to be available on this node * or it has to be in downloading state. @@ -356,7 +402,9 @@ public: /** * @brief Request remote files search * @jsonapi{development} - * @param[in] matchString string to look for in the search + * @param[in] matchString string to look for in the search. If files deep + * indexing is enabled at compile time support advanced features described + * at https://xapian.org/docs/queryparser.html * @param multiCallback function that will be called each time a search * result is received * @param[in] maxWait maximum wait time in seconds for search results @@ -364,7 +412,7 @@ public: */ virtual bool turtleSearchRequest( const std::string& matchString, - const std::function& results)>& multiCallback, + const std::function& results)>& multiCallback, rstime_t maxWait = 300 ) = 0; virtual TurtleRequestId turtleSearch(const std::string& string_to_match) = 0; @@ -627,8 +675,19 @@ public: */ virtual bool removeSharedDirectory(std::string dir) = 0; - virtual bool getIgnoreLists(std::list& ignored_prefixes, std::list& ignored_suffixes,uint32_t& flags) =0; - virtual void setIgnoreLists(const std::list& ignored_prefixes, const std::list& ignored_suffixes,uint32_t flags) =0; + /** + * @brief Get list of ignored file name prefixes and suffixes + * @param[out] ignoredPrefixes storage for ingored prefixes + * @param[out] ignoredSuffixes storage for ingored suffixes + * @param flags RS_FILE_SHARE_FLAGS_IGNORE_* + * @return false if something failed, true otherwhise + */ + virtual bool getIgnoreLists( + std::list& ignoredPrefixes, + std::list& ignoredSuffixes, + uint32_t& flags ) = 0; + + virtual void setIgnoreLists(const std::list& ignored_prefixes, const std::list& ignored_suffixes,uint32_t flags) =0; virtual void setWatchPeriod(int minutes) =0; virtual void setWatchEnabled(bool b) =0; diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 82a209f18..4b82792f2 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -47,7 +47,7 @@ extern RsGxsChannels* rsGxsChannels; struct RsGxsChannelGroup : RsSerializable { - RsGxsChannelGroup() : mAutoDownload(false) {} + RsGxsChannelGroup() : mAutoDownload(false) {} RsGroupMetaData mMeta; std::string mDescription; @@ -56,14 +56,17 @@ struct RsGxsChannelGroup : RsSerializable bool mAutoDownload; /// @see RsSerializable - virtual void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx ) + virtual void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override { RS_SERIAL_PROCESS(mMeta); RS_SERIAL_PROCESS(mDescription); RS_SERIAL_PROCESS(mImage); RS_SERIAL_PROCESS(mAutoDownload); } + + ~RsGxsChannelGroup() override; }; struct RsGxsChannelPost : RsSerializable @@ -82,8 +85,9 @@ struct RsGxsChannelPost : RsSerializable RsGxsImage mThumbnail; /// @see RsSerializable - virtual void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx ) + virtual void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override { RS_SERIAL_PROCESS(mMeta); RS_SERIAL_PROCESS(mOlderVersions); @@ -94,6 +98,8 @@ struct RsGxsChannelPost : RsSerializable RS_SERIAL_PROCESS(mSize); RS_SERIAL_PROCESS(mThumbnail); } + + ~RsGxsChannelPost() override; }; @@ -544,5 +550,5 @@ public: virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; ////////////////////////////////////////////////////////////////////////////// - virtual ~RsGxsChannels(); + ~RsGxsChannels() override; }; diff --git a/libretroshare/src/retroshare/rsgxstrans.h b/libretroshare/src/retroshare/rsgxstrans.h index a491f7bfe..e1b8d4d8f 100644 --- a/libretroshare/src/retroshare/rsgxstrans.h +++ b/libretroshare/src/retroshare/rsgxstrans.h @@ -23,9 +23,12 @@ #include "retroshare/rstokenservice.h" #include "retroshare/rsgxsifacehelper.h" #include "retroshare/rsgxscommon.h" +#include "rsitems/rsserviceids.h" -/// Subservices identifiers (like port for TCP) -enum class GxsTransSubServices : uint16_t +/** Subservices identifiers (like port for TCP) + * @deprecated pay special attention fixing this as it may break + * retro-compatibility */ +enum class RS_DEPRECATED_FOR(RsServiceType) GxsTransSubServices : uint16_t { UNKNOWN = 0x00, TEST_SERVICE = 0x01, diff --git a/libretroshare/src/retroshare/rsidentity.h b/libretroshare/src/retroshare/rsidentity.h index 5abdfb47b..cc5772e83 100644 --- a/libretroshare/src/retroshare/rsidentity.h +++ b/libretroshare/src/retroshare/rsidentity.h @@ -338,7 +338,7 @@ struct RsIdentityDetails : RsSerializable RS_SERIAL_PROCESS(mNickname); RS_SERIAL_PROCESS(mFlags); RS_SERIAL_PROCESS(mPgpId); - //RS_SERIAL_PROCESS(mReputation); + RS_SERIAL_PROCESS(mReputation); RS_SERIAL_PROCESS(mAvatar); RS_SERIAL_PROCESS(mPublishTS); RS_SERIAL_PROCESS(mLastUsageTS); @@ -352,7 +352,6 @@ struct RsIdentityDetails : RsSerializable struct RsIdentity : RsGxsIfaceHelper { explicit RsIdentity(RsGxsIface& gxs) : RsGxsIfaceHelper(gxs) {} - virtual ~RsIdentity() {} /** * @brief Create a new identity @@ -419,6 +418,14 @@ struct RsIdentity : RsGxsIfaceHelper */ virtual bool getOwnPseudonimousIds(std::vector& ids) = 0; + /** + * @brief Check if an id is known + * @jsonapi{development} + * @param[in] id Id to check + * @return true if the id is known, false otherwise + */ + virtual bool isKnownId(const RsGxsId& id) = 0; + /** * @brief Check if an id is own * @jsonapi{development} @@ -568,4 +575,6 @@ struct RsIdentity : RsGxsIfaceHelper RS_DEPRECATED_FOR("getIdentitiesSummaries getIdentitiesInfo") virtual bool getGroupData( const uint32_t& token, std::vector& groups) = 0; + + virtual ~RsIdentity(); }; diff --git a/libretroshare/src/retroshare/rsmsgs.h b/libretroshare/src/retroshare/rsmsgs.h index f56312659..109084dd7 100644 --- a/libretroshare/src/retroshare/rsmsgs.h +++ b/libretroshare/src/retroshare/rsmsgs.h @@ -3,7 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2007-2008 by Robert Fernie * + * Copyright (C) 2007-2008 Robert Fernie * + * Copyright (C) 2016-2019 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -19,8 +20,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RS_MSG_GUI_INTERFACE_H -#define RS_MSG_GUI_INTERFACE_H +#pragma once #include #include @@ -28,8 +28,11 @@ #include #include -#include "rstypes.h" -#include "rsgxsifacetypes.h" +#include "retroshare/rstypes.h" +#include "retroshare/rsgxsifacetypes.h" +#include "retroshare/rsevents.h" +#include "util/rsdeprecate.h" +#include "util/rsmemory.h" /********************** For Messages and Channels *****************/ @@ -93,9 +96,31 @@ const ChatLobbyFlags RS_CHAT_LOBBY_FLAGS_PGP_SIGNED ( 0x00000010 ) ; // requi typedef uint64_t ChatLobbyId ; typedef uint64_t ChatLobbyMsgId ; typedef std::string ChatLobbyNickName ; -typedef std::string RsMailMessageId; // should be uint32_t !! -typedef uint64_t MessageId ; +typedef std::string RsMailMessageId; // TODO: rebase on t_RsGenericIdType +/** + * Used to return a tracker id so the API user can keep track of sent mail + * status, it contains mail id, and recipient id + */ +struct RsMailIdRecipientIdPair : RsSerializable +{ + RsMailIdRecipientIdPair(RsMailMessageId mailId, RsGxsId recipientId): + mMailId(mailId), mRecipientId(recipientId) {} + + RsMailMessageId mMailId; + RsGxsId mRecipientId; + + /// @see RsSerializable + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext &ctx ) override; + + bool operator<(const RsMailIdRecipientIdPair& other) const; + bool operator==(const RsMailIdRecipientIdPair& other) const; + + RsMailIdRecipientIdPair() = default; + ~RsMailIdRecipientIdPair() override = default; +}; namespace Rs { @@ -137,38 +162,9 @@ class MsgAddress std::string _addr_string ; }; -class MessageInfo_v2 -{ - public: - //MessageInfo_v2() {} - - unsigned int msgflags; - - //RsMessageId msgId; - MsgAddress from ; - - std::list rcpt ; - - // Headers - // - std::string subject; - std::string msg; - rstime_t time_stamp ; - - //std::list headers ; - - std::string attach_title; - std::string attach_comment; - std::list files; - - int size; /* total of files */ - int count; /* file count */ -}; - struct MessageInfo : RsSerializable { MessageInfo(): msgflags(0), size(0), count(0), ts(0) {} - virtual ~MessageInfo() = default; std::string msgId; @@ -202,7 +198,10 @@ struct MessageInfo : RsSerializable int ts; // RsSerializable interface - void serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext &ctx) { + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext &ctx ) override + { RS_SERIAL_PROCESS(msgId); RS_SERIAL_PROCESS(rspeerid_srcId); @@ -230,26 +229,30 @@ struct MessageInfo : RsSerializable RS_SERIAL_PROCESS(ts); } + + ~MessageInfo() override; }; struct MsgInfoSummary : RsSerializable { MsgInfoSummary() : msgflags(0), count(0), ts(0) {} - virtual ~MsgInfoSummary() = default; - RsMailMessageId msgId; + RsMailMessageId msgId; RsPeerId srcId; uint32_t msgflags; - std::list msgtags; // that leaves 25 bits for user-defined tags. + std::list msgtags; /// that leaves 25 bits for user-defined tags. std::string title; - int count; /* file count */ + int count; /** file count */ rstime_t ts; - // RsSerializable interface - void serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext &ctx) { + /// @see RsSerializable + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext &ctx) override + { RS_SERIAL_PROCESS(msgId); RS_SERIAL_PROCESS(srcId); @@ -260,6 +263,8 @@ struct MsgInfoSummary : RsSerializable RS_SERIAL_PROCESS(count); RS_SERIAL_PROCESS(ts); } + + ~MsgInfoSummary() override; }; struct MsgTagInfo : RsSerializable @@ -291,6 +296,22 @@ struct MsgTagType : RsSerializable } //namespace Rs } //namespace Msgs +struct RsMailStatusEvent : RsEvent +{ + RsMailStatusEvent() : RsEvent(RsEventType::MAIL_STATUS_CHANGE) {} + + std::set mChangedMsgIds; + + /// @see RsEvent + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx) override + { + RsEvent::serial_process(j, ctx); + RS_SERIAL_PROCESS(mChangedMsgIds); + } + + ~RsMailStatusEvent() override; +}; #define RS_CHAT_PUBLIC 0x0001 #define RS_CHAT_PRIVATE 0x0002 @@ -322,10 +343,13 @@ struct MsgTagType : RsSerializable struct DistantChatPeerInfo { + DistantChatPeerInfo() : status(0),pending_items(0) {} + RsGxsId to_id ; RsGxsId own_id ; DistantChatPeerId peer_id ; // this is the tunnel id actually - uint32_t status ; // see the values in rsmsgs.h + uint32_t status ; // see the values in rsmsgs.h + uint32_t pending_items; // items not sent, waiting for a tunnel }; // Identifier for an chat endpoint like @@ -426,11 +450,10 @@ public: } }; -class VisibleChatLobbyRecord : RsSerializable +struct VisibleChatLobbyRecord : RsSerializable { -public: - VisibleChatLobbyRecord(): lobby_id(0), total_number_of_peers(0), last_report_time(0){} - virtual ~VisibleChatLobbyRecord() = default; + VisibleChatLobbyRecord(): + lobby_id(0), total_number_of_peers(0), last_report_time(0) {} ChatLobbyId lobby_id ; // unique id of the lobby std::string lobby_name ; // name to use for this lobby @@ -441,9 +464,11 @@ public: rstime_t last_report_time ; // last time the lobby was reported. ChatLobbyFlags lobby_flags ; // see RS_CHAT_LOBBY_PRIVACY_LEVEL_PUBLIC / RS_CHAT_LOBBY_PRIVACY_LEVEL_PRIVATE - // RsSerializable interface -public: - void serial_process(RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext &ctx) { + /// @see RsSerializable + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext &ctx) override + { RS_SERIAL_PROCESS(lobby_id); RS_SERIAL_PROCESS(lobby_name); RS_SERIAL_PROCESS(lobby_topic); @@ -453,6 +478,8 @@ public: RS_SERIAL_PROCESS(last_report_time); RS_SERIAL_PROCESS(lobby_flags); } + + ~VisibleChatLobbyRecord() override; }; class ChatLobbyInfo : RsSerializable @@ -499,13 +526,6 @@ class RsMsgs { public: - RsMsgs() {} - virtual ~RsMsgs() = default; - - /****************************************/ - /* Message Items */ - /****************************************/ - /** * @brief getMessageSummaries * @jsonapi{development} @@ -523,6 +543,33 @@ public: */ virtual bool getMessage(const std::string &msgId, Rs::Msgs::MessageInfo &msg) = 0; + /** + * @brief sendMail + * @jsonapi{development} + * @param[in] from GXS id of the author + * @param[in] subject Mail subject + * @param[in] mailBody Mail body + * @param[in] to list of To: recipients + * @param[in] cc list of CC: recipients + * @param[in] bcc list of BCC: recipients + * @param[in] attachments list of suggested files + * @param[out] trackingIds storage for tracking ids for each sent mail + * @param[out] errorMsg error message if errors occurred, empty otherwise + * @return number of successfully sent mails + */ + virtual uint32_t sendMail( + const RsGxsId from, + const std::string& subject, + const std::string& mailBody, + const std::set& to = std::set(), + const std::set& cc = std::set(), + const std::set& bcc = std::set(), + const std::vector& attachments = std::vector(), + std::set& trackingIds = + RS_DEFAULT_STORAGE_PARAM(std::set), + std::string& errorMsg = + RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0; + /** * @brief getMessageCount * @jsonapi{development} @@ -535,14 +582,6 @@ public: */ virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox) = 0; - /** - * @brief MessageSend - * @jsonapi{development} - * @param[in] info - * @return always true - */ - virtual bool MessageSend(Rs::Msgs::MessageInfo &info) = 0; - /** * @brief SystemMessage * @jsonapi{development} @@ -916,7 +955,14 @@ virtual bool initiateDistantChatConnexion( virtual bool getDistantChatStatus(const DistantChatPeerId& pid,DistantChatPeerInfo& info)=0; virtual bool closeDistantChatConnexion(const DistantChatPeerId& pid)=0; + /** + * @brief MessageSend + * @jsonapi{development} + * @param[in] info + * @return always true + */ + RS_DEPRECATED_FOR(sendMail) + virtual bool MessageSend(Rs::Msgs::MessageInfo &info) = 0; + + virtual ~RsMsgs(); }; - -#endif - diff --git a/libretroshare/src/retroshare/rsreputations.h b/libretroshare/src/retroshare/rsreputations.h index 8ea6ff213..b8426e642 100644 --- a/libretroshare/src/retroshare/rsreputations.h +++ b/libretroshare/src/retroshare/rsreputations.h @@ -72,7 +72,6 @@ struct RsReputationInfo : RsSerializable mFriendsNegativeVotes(0), mFriendAverageScore(RS_REPUTATION_THRESHOLD_DEFAULT), mOverallReputationLevel(RsReputationLevel::NEUTRAL) {} - virtual ~RsReputationInfo() {} RsOpinion mOwnOpinion; @@ -94,6 +93,8 @@ struct RsReputationInfo : RsSerializable RS_SERIAL_PROCESS(mFriendAverageScore); RS_SERIAL_PROCESS(mOverallReputationLevel); } + + virtual ~RsReputationInfo(); }; diff --git a/libretroshare/src/retroshare/rsturtle.h b/libretroshare/src/retroshare/rsturtle.h index 6f5ac469c..76120e85b 100644 --- a/libretroshare/src/retroshare/rsturtle.h +++ b/libretroshare/src/retroshare/rsturtle.h @@ -44,12 +44,10 @@ extern RsTurtle* rsTurtle; typedef uint32_t TurtleRequestId ; typedef RsPeerId TurtleVirtualPeerId; -/** - * This is the structure used to send back results of the turtle search, - * to other peers, to the notifyBase class, to the search caller or to the GUI. - */ struct TurtleFileInfo : RsSerializable { + TurtleFileInfo() : size(0) {} + uint64_t size; /// File size RsFileHash hash; /// File hash std::string name; /// File name @@ -65,7 +63,7 @@ struct TurtleFileInfo : RsSerializable RsTypeSerializer::serial_process( j, ctx, TLV_TYPE_STR_NAME, name, "name" ); } -}; +} RS_DEPRECATED_FOR(TurtleFileInfoV2); struct TurtleTunnelRequestDisplayInfo { @@ -120,10 +118,9 @@ public: virtual void setSessionEnabled(bool) = 0 ; virtual bool sessionEnabled() const = 0 ; - // Lauches a search request through the pipes, and immediately returns - // the request id, which will be further used by the gui to store results - // as they come back. - // + /** Lauches a search request through the pipes, and immediately returns + * the request id, which will be further used by client services to + * handle results as they come back. */ virtual TurtleRequestId turtleSearch( unsigned char *search_bin_data, uint32_t search_bin_data_len, RsTurtleClientService* client_service ) = 0; diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 52495b664..1f5107fe3 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -399,11 +399,20 @@ struct FileChunksInfo : RsSerializable CHUNK_STRATEGY_PROGRESSIVE }; - struct SliceInfo + struct SliceInfo : RsSerializable { uint32_t start; uint32_t size; RsPeerId peer_id; + + /// @see RsSerializable + void serial_process(RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx) override + { + RS_SERIAL_PROCESS(start); + RS_SERIAL_PROCESS(size); + RS_SERIAL_PROCESS(peer_id); + } }; uint64_t file_size; /// real size of the file @@ -432,7 +441,7 @@ struct FileChunksInfo : RsSerializable RS_SERIAL_PROCESS(chunks); RS_SERIAL_PROCESS(compressed_peer_availability_maps); RS_SERIAL_PROCESS(active_chunks); - //RS_SERIAL_PROCESS(pending_slices); + RS_SERIAL_PROCESS(pending_slices); } }; diff --git a/libretroshare/src/rs_upnp/upnphandler_linux.cc b/libretroshare/src/rs_upnp/upnphandler_libupnp.cc similarity index 98% rename from libretroshare/src/rs_upnp/upnphandler_linux.cc rename to libretroshare/src/rs_upnp/upnphandler_libupnp.cc index df9bc66e9..17e096a08 100644 --- a/libretroshare/src/rs_upnp/upnphandler_linux.cc +++ b/libretroshare/src/rs_upnp/upnphandler_libupnp.cc @@ -1,5 +1,5 @@ /******************************************************************************* - * libretroshare/src/upnp: upnphandler_linux.cc * + * libretroshare/src/upnp: upnphandler_libupnp.cc * * * * libretroshare: retroshare core library * * * @@ -28,7 +28,7 @@ extern "C" { #endif /* This stuff is actually C */ -#include "rs_upnp/upnphandler_linux.h" +#include "rs_upnp/upnphandler_libupnp.h" #include "util/rsnet.h" diff --git a/libretroshare/src/rs_upnp/upnphandler_linux.h b/libretroshare/src/rs_upnp/upnphandler_libupnp.h similarity index 95% rename from libretroshare/src/rs_upnp/upnphandler_linux.h rename to libretroshare/src/rs_upnp/upnphandler_libupnp.h index dc5207a7c..bc04d6d36 100644 --- a/libretroshare/src/rs_upnp/upnphandler_linux.h +++ b/libretroshare/src/rs_upnp/upnphandler_libupnp.h @@ -1,5 +1,5 @@ /******************************************************************************* - * libretroshare/src/upnp: upnphandler_linux.h * + * libretroshare/src/upnp: upnphandler_libupnp.h * * * * libretroshare: retroshare core library * * * @@ -19,8 +19,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef _RS_UPNP_IFACE_H -#define _RS_UPNP_IFACE_H +#pragma once #include @@ -99,5 +98,3 @@ class upnphandler: public pqiNetAssistFirewall /* info from upnp */ int CtrlPointCallbackEventHandler(Upnp_EventType ,void* , void*); - -#endif /* _RS_UPNP_IFACE_H */ diff --git a/libretroshare/src/rs_upnp/upnphandler_miniupnp.h b/libretroshare/src/rs_upnp/upnphandler_miniupnp.h index e1616a0b9..d07304724 100644 --- a/libretroshare/src/rs_upnp/upnphandler_miniupnp.h +++ b/libretroshare/src/rs_upnp/upnphandler_miniupnp.h @@ -19,12 +19,9 @@ * along with this program. If not, see . * * * *******************************************************************************/ -//windows/osx (miniupnpc) implementation -#ifndef _RS_UPNP_IFACE_H -#define _RS_UPNP_IFACE_H +#pragma once #include - #include #include @@ -127,5 +124,3 @@ bool checkUPnPActive(); std::list activeForwards; }; - -#endif /* _RS_UPNP_IFACE_H */ diff --git a/libretroshare/src/rsserver/p3msgs.cc b/libretroshare/src/rsserver/p3msgs.cc index 3f987af49..ad74dbdde 100644 --- a/libretroshare/src/rsserver/p3msgs.cc +++ b/libretroshare/src/rsserver/p3msgs.cc @@ -3,7 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2004-2006 by Robert Fernie * + * Copyright (C) 2004-2006 Robert Fernie * + * Copyright (C) 2016-2019 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -20,6 +21,7 @@ * * *******************************************************************************/ #include +#include #include "util/rsdir.h" #include "util/rsdebug.h" @@ -38,8 +40,7 @@ using namespace Rs::Msgs; -/* external reference point */ -RsMsgs *rsMsgs = NULL; +/*extern*/ RsMsgs* rsMsgs = nullptr; /****************************************/ /****************************************/ @@ -300,6 +301,22 @@ bool p3Msgs::MessageSend(MessageInfo &info) return mMsgSrv->MessageSend(info); } +uint32_t p3Msgs::sendMail( + const RsGxsId from, + const std::string& subject, + const std::string& body, + const std::set& to, + const std::set& cc, + const std::set& bcc, + const std::vector& attachments, + std::set& trackingIds, + std::string& errorMsg ) +{ + return mMsgSrv->sendMail( + from, subject, body, to, cc, bcc, attachments, + trackingIds, errorMsg ); +} + bool p3Msgs::SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag) { return mMsgSrv->SystemMessage(title, message, systemFlag); @@ -541,3 +558,28 @@ uint32_t p3Msgs::getDistantChatPermissionFlags() return mChatSrv->getDistantChatPermissionFlags() ; } +RsMsgs::~RsMsgs() = default; +RsMailStatusEvent::~RsMailStatusEvent() = default; +Rs::Msgs::MessageInfo::~MessageInfo() = default; +MsgInfoSummary::~MsgInfoSummary() = default; +VisibleChatLobbyRecord::~VisibleChatLobbyRecord() = default; + +void RsMailIdRecipientIdPair::serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) +{ + RS_SERIAL_PROCESS(mMailId); + RS_SERIAL_PROCESS(mRecipientId); +} + +bool RsMailIdRecipientIdPair::operator<(const RsMailIdRecipientIdPair& o) const +{ + return std::tie( mMailId, mRecipientId) < + std::tie(o.mMailId, o.mRecipientId); +} + +bool RsMailIdRecipientIdPair::operator==(const RsMailIdRecipientIdPair& o) const +{ + return std::tie( mMailId, mRecipientId) == + std::tie(o.mMailId, o.mRecipientId); +} diff --git a/libretroshare/src/rsserver/p3msgs.h b/libretroshare/src/rsserver/p3msgs.h index b567ab48f..955018d26 100644 --- a/libretroshare/src/rsserver/p3msgs.h +++ b/libretroshare/src/rsserver/p3msgs.h @@ -3,7 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2007-2008 by Robert Fernie * + * Copyright (C) 2007-2008 Robert Fernie * + * Copyright (C) 2016-2019 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -19,8 +20,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RS_P3MSG_INTERFACE_H -#define RS_P3MSG_INTERFACE_H +#pragma once #include "retroshare/rsmsgs.h" #include "retroshare/rsgxsifacetypes.h" @@ -38,11 +38,25 @@ class RsChatMsgItem; */ class p3Msgs: public RsMsgs { - public: +public: - p3Msgs(p3MsgService *p3m, p3ChatService *p3c) - :mMsgSrv(p3m), mChatSrv(p3c) { return; } - virtual ~p3Msgs() { return; } + p3Msgs(p3MsgService *p3m, p3ChatService *p3c) : + mMsgSrv(p3m), mChatSrv(p3c) {} + ~p3Msgs() override = default; + + /// @see RsMsgs + uint32_t sendMail( + const RsGxsId from, + const std::string& subject, + const std::string& body, + const std::set& to = std::set(), + const std::set& cc = std::set(), + const std::set& bcc = std::set(), + const std::vector& attachments = std::vector(), + std::set& trackingIds = + RS_DEFAULT_STORAGE_PARAM(std::set), + std::string& errorMsg = + RS_DEFAULT_STORAGE_PARAM(std::string) ) override; /****************************************/ /* Message Items */ @@ -54,6 +68,7 @@ class p3Msgs: public RsMsgs virtual bool getMessage(const std::string &mId, Rs::Msgs::MessageInfo &msg); virtual void getMessageCount(uint32_t &nInbox, uint32_t &nInboxNew, uint32_t &nOutbox, uint32_t &nDraftbox, uint32_t &nSentbox, uint32_t &nTrashbox); + RS_DEPRECATED_FOR(sendMail) virtual bool MessageSend(Rs::Msgs::MessageInfo &info); virtual bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag); virtual bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId); @@ -172,6 +187,3 @@ class p3Msgs: public RsMsgs p3MsgService *mMsgSrv; p3ChatService *mChatSrv; }; - - -#endif diff --git a/libretroshare/src/rsserver/p3serverconfig.cc b/libretroshare/src/rsserver/p3serverconfig.cc index 4e22c3a4d..f31529fa3 100644 --- a/libretroshare/src/rsserver/p3serverconfig.cc +++ b/libretroshare/src/rsserver/p3serverconfig.cc @@ -160,7 +160,6 @@ int p3ServerConfig::getConfigNetStatus(RsConfigNetStatus &status) pqiNetStatus nstatus; mNetMgr->getNetStatus(nstatus); - status.netLocalOk = nstatus.mLocalAddrOk; status.netUpnpOk = nstatus.mUpnpOk; status.netStunOk = false; status.netExtAddressOk = nstatus.mExtAddrOk; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 3b1f172cf..cbf2ff1c9 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -702,7 +702,7 @@ RsGRouter *rsGRouter = NULL ; #include "util/rsrandom.h" #ifdef RS_USE_LIBUPNP -# include "rs_upnp/upnphandler_linux.h" +# include "rs_upnp/upnphandler_libupnp.h" #else // def RS_USE_LIBUPNP # include "rs_upnp/upnphandler_miniupnp.h" #endif // def RS_USE_LIBUPNP diff --git a/libretroshare/src/serialiser/rsserializer.cc b/libretroshare/src/serialiser/rsserializer.cc index c1385fddf..71d7abea3 100644 --- a/libretroshare/src/serialiser/rsserializer.cc +++ b/libretroshare/src/serialiser/rsserializer.cc @@ -27,6 +27,7 @@ #include "serialiser/rsserializer.h" #include "serialiser/rstypeserializer.h" #include "util/stacktrace.h" +#include "util/rsdebug.h" const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_NONE ( 0x0000 ); const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_CONFIG ( 0x0001 ); @@ -36,6 +37,16 @@ const SerializationFlags RsGenericSerializer::SERIALIZATION_FLAG_YIELDING ( 0 RsItem *RsServiceSerializer::deserialise(void *data, uint32_t *size) { + if(!data || !size || !*size) + { + RsErr() << __PRETTY_FUNCTION__ << " Called with null paramethers data: " + << data << " size: " << static_cast(size) << " *size: " + << (size ? *size : 0) << " this should never happen!" + << std::endl; + print_stacktrace(); + return nullptr; + } + if(mFlags & SERIALIZATION_FLAG_SKIP_HEADER) { std::cerr << "(EE) Cannot deserialise item with flags SERIALIZATION_FLAG_SKIP_HEADER. Check your code!" << std::endl; diff --git a/libretroshare/src/services/broadcastdiscoveryservice.cc b/libretroshare/src/services/broadcastdiscoveryservice.cc index 526610ced..d49e04a17 100644 --- a/libretroshare/src/services/broadcastdiscoveryservice.cc +++ b/libretroshare/src/services/broadcastdiscoveryservice.cc @@ -25,6 +25,10 @@ #include #include +#ifdef __ANDROID__ +# include +#endif // def __ANDROID__ + #include "services/broadcastdiscoveryservice.h" #include "retroshare/rspeers.h" #include "serialiser/rsserializable.h" @@ -94,6 +98,12 @@ BroadcastDiscoveryService::BroadcastDiscoveryService( { if(mRsPeers.isHiddenNode(mRsPeers.getOwnId())) return; +#ifdef __ANDROID__ + createMulticastLock(); +#endif // def __ANDROID__ + + enableMulticastListening(); + mUdcParameters.set_can_discover(true); mUdcParameters.set_can_be_discovered(true); mUdcParameters.set_port(port); @@ -104,7 +114,10 @@ BroadcastDiscoveryService::BroadcastDiscoveryService( } BroadcastDiscoveryService::~BroadcastDiscoveryService() -{ mUdcPeer.Stop(true); } +{ + mUdcPeer.Stop(true); + disableMulticastListening(); +} std::vector BroadcastDiscoveryService::getDiscoveredPeers() @@ -192,6 +205,7 @@ RsBroadcastDiscoveryResult BroadcastDiscoveryService::createResult( BroadcastDiscoveryPack::fromSerializedString(uData); RsBroadcastDiscoveryResult rbdr; + rbdr.mPgpFingerprint = bdp.mPgpFingerprint; rbdr.mSslId = bdp.mSslId; rbdr.mProfileName = bdp.mProfileName; rbdr.mLocator. @@ -202,6 +216,95 @@ RsBroadcastDiscoveryResult BroadcastDiscoveryService::createResult( return rbdr; } +bool BroadcastDiscoveryService::isMulticastListeningEnabled() +{ +#ifdef __ANDROID__ + return assertMulticastLockIsvalid() && + mWifiMulticastLock.callMethod("isHeld"); +#endif // def __ANDROID__ + + return true; +} + +bool BroadcastDiscoveryService::enableMulticastListening() +{ +#ifdef __ANDROID__ + if(assertMulticastLockIsvalid() && !isMulticastListeningEnabled()) + { + mWifiMulticastLock.callMethod("acquire"); + return true; + } +#endif // def __ANDROID__ + + return false; +} + +bool BroadcastDiscoveryService::disableMulticastListening() +{ +#ifdef __ANDROID__ + if(assertMulticastLockIsvalid() && isMulticastListeningEnabled()) + { + mWifiMulticastLock.callMethod("release"); + return true; + } +#endif // def __ANDROID__ + + return false; +} + +#ifdef __ANDROID__ +bool BroadcastDiscoveryService::createMulticastLock() +{ + Dbg2() << __PRETTY_FUNCTION__ << std::endl; + + constexpr auto fname = __PRETTY_FUNCTION__; + const auto failure = [&](const std::string& err) + { + RsErr() << fname << " " << err << std::endl; + return false; + }; + + if(mWifiMulticastLock.isValid()) + return failure("mWifiMulticastLock is already initialized"); + + QAndroidJniObject context = QtAndroid::androidContext(); + if(!context.isValid()) + return failure("Cannot retrieve Android context"); + + QAndroidJniObject WIFI_SERVICE = QAndroidJniObject::getStaticObjectField( + "android.content.Context", "WIFI_SERVICE", "Ljava/lang/String;"); + if(!WIFI_SERVICE.isValid()) + return failure("Cannot retrieve Context.WIFI_SERVICE value"); + + QAndroidJniObject wifiManager = context.callObjectMethod( + "getSystemService", "(Ljava/lang/String;)Ljava/lang/Object;", + WIFI_SERVICE.object() ); + if(!wifiManager.isValid()) + return failure("Cannot retrieve Android Wifi Manager"); + + mWifiMulticastLock = wifiManager.callObjectMethod( + "createMulticastLock", + "(Ljava/lang/String;)Landroid/net/wifi/WifiManager$MulticastLock;", + QAndroidJniObject::fromString(fname).object() ); + if(!mWifiMulticastLock.isValid()) + return failure("Cannot create WifiManager.MulticastLock"); + + return true; +} + +bool BroadcastDiscoveryService::assertMulticastLockIsvalid() +{ + if(!mWifiMulticastLock.isValid()) + { + RsErr() << __PRETTY_FUNCTION__ << " mWifiMulticastLock is invalid!" + << std::endl; + print_stacktrace(); + return false; + } + return true; +} +#endif // def __ANDROID__ + RsBroadcastDiscovery::~RsBroadcastDiscovery() = default; RsBroadcastDiscoveryResult::~RsBroadcastDiscoveryResult() = default; RsBroadcastDiscoveryPeerFoundEvent::~RsBroadcastDiscoveryPeerFoundEvent() = default; diff --git a/libretroshare/src/services/broadcastdiscoveryservice.h b/libretroshare/src/services/broadcastdiscoveryservice.h index 17129c995..f7c52466a 100644 --- a/libretroshare/src/services/broadcastdiscoveryservice.h +++ b/libretroshare/src/services/broadcastdiscoveryservice.h @@ -27,8 +27,13 @@ #include +#ifdef __ANDROID__ +# include +#endif // def __ANDROID__ + #include "retroshare/rsbroadcastdiscovery.h" #include "util/rsthreads.h" +#include "util/rsdebug.h" namespace UDC = udpdiscovery; class RsPeers; @@ -37,13 +42,21 @@ class BroadcastDiscoveryService : public RsBroadcastDiscovery, public RsTickingThread { public: - // TODO: std::shared_ptr mRsPeers; BroadcastDiscoveryService(RsPeers& pRsPeers); - virtual ~BroadcastDiscoveryService() override; + ~BroadcastDiscoveryService() override; /// @see RsBroadcastDiscovery std::vector getDiscoveredPeers() override; + /// @see RsBroadcastDiscovery + bool isMulticastListeningEnabled() override; + + /// @see RsBroadcastDiscovery + bool enableMulticastListening() override; + + /// @see RsBroadcastDiscovery + bool disableMulticastListening() override; + /// @see RsTickingThread void data_tick() override; @@ -63,4 +76,22 @@ protected: RsBroadcastDiscoveryResult createResult( const UDC::IpPort& ipp, const std::string& uData ); + +#ifdef __ANDROID__ + /** Android WifiManager.MulticastLock */ + QAndroidJniObject mWifiMulticastLock; + + /** Initialize the wifi multicast lock without acquiring it + * Needed to enable multicast listening in Android, for RetroShare broadcast + * discovery inspired by: + * https://github.com/flutter/flutter/issues/16335#issuecomment-420547860 + */ + bool createMulticastLock(); + + /** Return false if mWifiMulticastLock is invalid and print error messages */ + bool assertMulticastLockIsvalid(); + +#endif // def __ANDROID__ + + RS_SET_CONTEXT_DEBUG_LEVEL(3) }; diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index d25d159df..70e0763bf 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -44,9 +44,9 @@ #include "util/rsrandom.h" #include "util/rsstring.h" -#ifdef RS_DEEP_SEARCH -# include "deep_search/deep_search.h" -#endif // RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX +# include "deep_search/channelsindex.hpp" +#endif // RS_DEEP_CHANNEL_INDEX /**** @@ -1149,9 +1149,9 @@ bool p3GxsChannels::createChannelV2( channelId = channel.mMeta.mGroupId; -#ifdef RS_DEEP_SEARCH - DeepSearch::indexChannelGroup(channel); -#endif // RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX + DeepChannelsIndex::indexChannelGroup(channel); +#endif // RS_DEEP_CHANNEL_INDEX return true; } @@ -1180,9 +1180,9 @@ bool p3GxsChannels::createChannel(RsGxsChannelGroup& channel) return false; } -#ifdef RS_DEEP_SEARCH - DeepSearch::indexChannelGroup(channel); -#endif // RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX + DeepChannelsIndex::indexChannelGroup(channel); +#endif // RS_DEEP_CHANNEL_INDEX return true; } @@ -1333,9 +1333,9 @@ bool p3GxsChannels::editChannel(RsGxsChannelGroup& channel) return false; } -#ifdef RS_DEEP_SEARCH - DeepSearch::indexChannelGroup(channel); -#endif // RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX + DeepChannelsIndex::indexChannelGroup(channel); +#endif // RS_DEEP_CHANNEL_INDEX return true; } @@ -1401,9 +1401,9 @@ bool p3GxsChannels::createPostV2( if(RsGenExchange::getPublishedMsgMeta(token,post.mMeta)) { -#ifdef RS_DEEP_SEARCH - DeepSearch::indexChannelPost(post); -#endif // RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX + DeepChannelsIndex::indexChannelPost(post); +#endif // RS_DEEP_CHANNEL_INDEX postId = post.mMeta.mMsgId; return true; @@ -1787,9 +1787,9 @@ bool p3GxsChannels::createPost(RsGxsChannelPost& post) if(RsGenExchange::getPublishedMsgMeta(token,post.mMeta)) { -#ifdef RS_DEEP_SEARCH - DeepSearch::indexChannelPost(post); -#endif // RS_DEEP_SEARCH +#ifdef RS_DEEP_CHANNEL_INDEX + DeepChannelsIndex::indexChannelPost(post); +#endif // RS_DEEP_CHANNEL_INDEX return true; } @@ -2466,4 +2466,6 @@ void p3GxsChannels::cleanTimedOutCallbacks() } // RS_STACK_MUTEX(mDistantChannelsCallbacksMapMutex) } +RsGxsChannelGroup::~RsGxsChannelGroup() = default; +RsGxsChannelPost::~RsGxsChannelPost() = default; RsGxsChannels::~RsGxsChannels() = default; diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 95adf0b96..dc2d53cb3 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -810,9 +810,15 @@ bool p3IdService::getOwnIds(std::list &ownIds,bool signed_only) return true ; } +bool p3IdService::isKnownId(const RsGxsId& id) +{ + RS_STACK_MUTEX(mIdMtx); + return mKeyCache.is_cached(id) || + std::find(mOwnIds.begin(), mOwnIds.end(),id) != mOwnIds.end(); +} bool p3IdService::identityToBase64( const RsGxsId& id, - std::string& base64String ) + std::string& base64String ) { return serialiseIdentityToMemory(id, base64String); } bool p3IdService::serialiseIdentityToMemory( const RsGxsId& id, @@ -4788,3 +4794,7 @@ RsIdentityUsage::RsIdentityUsage( RsIdentityUsage::RsIdentityUsage() : mServiceId(RsServiceType::NONE), mUsageCode(UNKNOWN_USAGE), mAdditionalId(0) {} + +RsIdentity::~RsIdentity() = default; +RsReputationInfo::~RsReputationInfo() = default; +RsGixs::~RsGixs() = default; diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index 4e22698b6..0b8f433b6 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -293,12 +293,12 @@ public: /// @see RsIdentity bool getOwnPseudonimousIds(std::vector& ids) override; - virtual bool getOwnIds(std::list &ownIds, bool signed_only = false); + bool getOwnIds( + std::list &ownIds, bool signed_only = false ) override; - //virtual bool getPublicKey(const RsGxsId &id, RsTlvSecurityKey &key) ; - //virtual void networkRequestPublicKey(const RsGxsId& key_id,const std::list& peer_ids) ; + bool isKnownId(const RsGxsId& id) override; - virtual bool isOwnId(const RsGxsId& key_id) ; + bool isOwnId(const RsGxsId& key_id) override; virtual bool signData( const uint8_t* data, uint32_t data_size, @@ -619,7 +619,7 @@ private: bool ownIdsAreLoaded() { RS_STACK_MUTEX(mIdMtx); return mOwnIdsLoaded; } bool mAutoAddFriendsIdentitiesAsContacts; - uint32_t mMaxKeepKeysBanned ; + uint32_t mMaxKeepKeysBanned; RS_SET_CONTEXT_DEBUG_LEVEL(1) }; diff --git a/libretroshare/src/services/p3msgservice.cc b/libretroshare/src/services/p3msgservice.cc index 3e2f9902c..ebf2ce496 100644 --- a/libretroshare/src/services/p3msgservice.cc +++ b/libretroshare/src/services/p3msgservice.cc @@ -3,7 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2004-2008 Robert Fernie * + * Copyright (C) 2004-2008 Robert Fernie * + * Copyright (C) 2016-2019 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -55,19 +56,10 @@ #include #include -//#define MSG_DEBUG 1 -//#define DEBUG_DISTANT_MSG -//#define DISABLE_DISTANT_MESSAGES -//#define DEBUG_DISTANT_MSG - -typedef unsigned int uint; - using namespace Rs::Msgs; -static struct RsLog::logInfo msgservicezoneInfo = {RsLog::Default, "msgservice"}; -#define msgservicezone &msgservicezoneInfo - -static const uint32_t RS_MSG_DISTANT_MESSAGE_HASH_KEEP_TIME = 2*30*86400 ; // keep msg hashes for 2 months to avoid re-sent msgs +/// keep msg hashes for 2 months to avoid re-sent msgs +static constexpr uint32_t RS_MSG_DISTANT_MESSAGE_HASH_KEEP_TIME = 2*30*86400; /* Another little hack ..... unique message Ids * will be handled in this class..... @@ -128,11 +120,8 @@ uint32_t p3MsgService::getNewUniqueMsgId() return mMsgUniqueId++; } -int p3MsgService::tick() +int p3MsgService::tick() { - pqioutput(PQL_DEBUG_BASIC, msgservicezone, - "p3MsgService::tick()"); - /* don't worry about increasing tick rate! * (handled by p3service) */ @@ -158,7 +147,7 @@ void p3MsgService::cleanListOfReceivedMessageHashes() { RS_STACK_MUTEX(recentlyReceivedMutex); - rstime_t now = time(NULL); + rstime_t now = time(nullptr); for( auto it = mRecentlyReceivedMessageHashes.begin(); it != mRecentlyReceivedMessageHashes.end(); ) @@ -173,21 +162,13 @@ void p3MsgService::cleanListOfReceivedMessageHashes() else ++it; } -int p3MsgService::status() -{ - pqioutput(PQL_DEBUG_BASIC, msgservicezone, - "p3MsgService::status()"); - - return 1; -} - void p3MsgService::processIncomingMsg(RsMsgItem *mi) { - mi -> recvTime = time(NULL); + mi -> recvTime = static_cast(time(nullptr)); mi -> msgId = getNewUniqueMsgId(); { - RsStackMutex stack(mMsgMtx); /*** STACK LOCKED MTX ***/ + RS_STACK_MUTEX(mMsgMtx); /* from a peer */ @@ -226,6 +207,13 @@ void p3MsgService::processIncomingMsg(RsMsgItem *mi) } RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_ADD); + + if(rsEvents) + { + std::shared_ptr pEvent(new RsMailStatusEvent()); + pEvent->mChangedMsgIds.insert(std::to_string(mi->msgId)); + rsEvents->postEvent(pEvent); + } } bool p3MsgService::checkAndRebuildPartialMessage(RsMsgItem *ci) @@ -289,7 +277,8 @@ void p3MsgService::handleIncomingItem(RsMsgItem *mi) { bool changed = false ; - if(checkAndRebuildPartialMessage(mi)) // only returns true when a msg is complete. + // only returns true when a msg is complete. + if(checkAndRebuildPartialMessage(mi)) { processIncomingMsg(mi); changed = true ; @@ -356,6 +345,9 @@ int p3MsgService::checkOutgoingMessages() bool changed = false; std::list output_queue; + using Evt_t = RsMailStatusEvent; + std::shared_ptr pEvent(new Evt_t()); + { RS_STACK_MUTEX(mMsgMtx); /********** STACK LOCKED MTX ******/ @@ -386,12 +378,12 @@ int p3MsgService::checkOutgoingMessages() if(should_send) { - /* send msg */ - pqioutput( PQL_DEBUG_BASIC, msgservicezone, - "p3MsgService::checkOutGoingMessages() Sending out message"); + Dbg3() << __PRETTY_FUNCTION__ << " Sending out message" + << std::endl; /* remove the pending flag */ - output_queue.push_back(mit->second) ; + output_queue.push_back(mit->second); + pEvent->mChangedMsgIds.insert(std::to_string(mit->first)); /* When the message is a distant msg, dont remove it yet from * the list. Only mark it as being sent, so that we don't send @@ -414,8 +406,8 @@ int p3MsgService::checkOutgoingMessages() } else { - pqioutput( PQL_DEBUG_BASIC, msgservicezone, - "p3MsgService::checkOutGoingMessages() Delaying until available..."); + Dbg3() << __PRETTY_FUNCTION__ << " Delaying until available..." + << std::endl; } } @@ -446,6 +438,9 @@ int p3MsgService::checkOutgoingMessages() if(changed) RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_MOD); + if(rsEvents && !pEvent->mChangedMsgIds.empty()) + rsEvents->postEvent(pEvent); + return 0; } @@ -911,6 +906,8 @@ bool p3MsgService::removeMsgId(const std::string &mid) } bool changed = false; + using Evt_t = RsMailStatusEvent; + std::shared_ptr pEvent(new Evt_t()); { RsStackMutex stack(mMsgMtx); /********** STACK LOCKED MTX ******/ @@ -922,6 +919,7 @@ bool p3MsgService::removeMsgId(const std::string &mid) RsMsgItem *mi = mit->second; imsg.erase(mit); delete mi; + pEvent->mChangedMsgIds.insert(mid); } mit = msgOutgoing.find(msgId); @@ -931,6 +929,7 @@ bool p3MsgService::removeMsgId(const std::string &mid) RsMsgItem *mi = mit->second; msgOutgoing.erase(mit); delete mi; + pEvent->mChangedMsgIds.insert(mid); } std::map::iterator srcIt = mSrcIds.find(msgId); @@ -938,6 +937,7 @@ bool p3MsgService::removeMsgId(const std::string &mid) changed = true; delete (srcIt->second); mSrcIds.erase(srcIt); + pEvent->mChangedMsgIds.insert(mid); } } @@ -950,6 +950,9 @@ bool p3MsgService::removeMsgId(const std::string &mid) RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_MOD); } + if(rsEvents && !pEvent->mChangedMsgIds.empty()) + rsEvents->postEvent(pEvent); + return changed; } @@ -1095,12 +1098,14 @@ bool p3MsgService::setMsgParentId(uint32_t msgId, uint32_t msgParentId) /****************************************/ /****************************************/ /* Message Items */ -uint32_t p3MsgService::sendMessage(RsMsgItem *item) // no from field because it's implicitly our own PeerId +// no from field because it's implicitly our own PeerId +uint32_t p3MsgService::sendMessage(RsMsgItem* item) { - if(!item) - return 0 ; - - pqioutput(PQL_DEBUG_BASIC, msgservicezone, "p3MsgService::sendMessage()"); + if(!item) + { + RsErr() << __PRETTY_FUNCTION__ << " item can't be null" << std::endl; + return 0; + } item->msgId = getNewUniqueMsgId(); /* grabs Mtx as well */ item->msgFlags |= (RS_MSG_FLAGS_OUTGOING | RS_MSG_FLAGS_PENDING); /* add pending flag */ @@ -1131,7 +1136,12 @@ uint32_t p3MsgService::sendMessage(RsMsgItem *item) // no from field because uint32_t p3MsgService::sendDistantMessage(RsMsgItem *item, const RsGxsId& from) { - if(!item) return 0; + if(!item) + { + RsErr() << __PRETTY_FUNCTION__ << " item can't be null" << std::endl; + print_stacktrace(); + return 0; + } item->msgId = getNewUniqueMsgId(); /* grabs Mtx as well */ item->msgFlags |= ( RS_MSG_FLAGS_DISTANT | RS_MSG_FLAGS_OUTGOING | @@ -1178,8 +1188,6 @@ bool p3MsgService::MessageSend(MessageInfo &info) if (msg) { - std::list::iterator it ; - if (msg->msgFlags & RS_MSG_FLAGS_SIGNED) msg->msgFlags |= RS_MSG_FLAGS_SIGNATURE_CHECKS; // this is always true, since we are sending the message @@ -1192,14 +1200,125 @@ bool p3MsgService::MessageSend(MessageInfo &info) imsg[msg->msgId] = msg; RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_ADD); - // - // // return new message id - // rs_sprintf(info.msgId, "%lu", msg->msgId); } return true; } +uint32_t p3MsgService::sendMail( + const RsGxsId from, + const std::string& subject, + const std::string& body, + const std::set& to, + const std::set& cc, + const std::set& bcc, + const std::vector& attachments, + std::set& trackingIds, + std::string& errorMsg ) +{ + errorMsg.clear(); + const std::string fname = __PRETTY_FUNCTION__; + auto pCheck = [&](bool test, const std::string& errMsg) + { + if(!test) + { + errorMsg = errMsg; + RsErr() << fname << " " << errMsg << std::endl; + } + return test; + }; + + if(!pCheck(!from.isNull(), "from can't be null")) return false; + if(!pCheck( rsIdentity->isOwnId(from), + "from must be own identity") ) return false; + if(!pCheck(!(to.empty() && cc.empty() && bcc.empty()), + "You must specify at least one recipient" )) return false; + + auto dstCheck = + [&](const std::set& dstSet, const std::string& setName) + { + for(const RsGxsId& dst: dstSet) + { + if(dst.isNull()) + { + errorMsg = setName + " contains a null recipient"; + RsErr() << fname << " " << errorMsg << std::endl; + return false; + } + + if(!rsIdentity->isKnownId(dst)) + { + rsIdentity->requestIdentity(dst); + errorMsg = setName + " contains an unknown recipient: " + + dst.toStdString(); + RsErr() << fname << " " << errorMsg << std::endl; + return false; + } + } + return true; + }; + + if(!dstCheck(to, "to")) return false; + if(!dstCheck(cc, "cc")) return false; + if(!dstCheck(bcc, "bcc")) return false; + + MessageInfo msgInfo; + + msgInfo.rsgxsid_srcId = from; + msgInfo.title = subject; + msgInfo.msg = body; + msgInfo.rsgxsid_msgto = to; + msgInfo.rsgxsid_msgcc = cc; + msgInfo.rsgxsid_msgbcc = bcc; + std::copy( attachments.begin(), attachments.end(), + std::back_inserter(msgInfo.files) ); + + uint32_t ret = 0; + using Evt_t = RsMailStatusEvent; + std::shared_ptr pEvent(new Evt_t()); + + auto pSend = [&](const std::set& sDest) + { + for(const RsGxsId& dst : sDest) + { + RsMsgItem* msgItem = initMIRsMsg(msgInfo, dst); + if(!msgItem) + { + errorMsg += " initMIRsMsg from: " + from.toStdString() + + " dst: " + dst.toStdString() + " subject: " + subject + + " returned nullptr!\n"; + RsErr() << fname << errorMsg; + continue; + } + + uint32_t msgId = sendDistantMessage(msgItem, from); + // ensure we don't use that ptr again without noticing + msgItem = nullptr; + + if(!msgId) + { + errorMsg += " sendDistantMessage from: " + from.toStdString() + + " dst: " + dst.toStdString() + " subject: " + subject + + " returned 0!\n"; + RsErr() << fname << errorMsg; + continue; + } + + const RsMailMessageId mailId = std::to_string(msgId); + pEvent->mChangedMsgIds.insert(mailId); + trackingIds.insert(RsMailIdRecipientIdPair(mailId, dst)); + ++ret; + } + }; + + pSend(to); + pSend(cc); + pSend(bcc); + + if(rsEvents) rsEvents->postEvent(pEvent); + return ret; +} + bool p3MsgService::SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag) { if ((systemFlag & RS_MSG_SYSTEM) == 0) { @@ -1890,20 +2009,21 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id, { RS_STACK_MUTEX(mMsgMtx); - std::cerr << "(WW) p3MsgService::notifyDataStatus: Global router tells " - << "us that item ID " << id - << " could not be delivered on time."; - auto it = _ongoing_messages.find(id); if(it == _ongoing_messages.end()) { - std::cerr << " (EE) cannot find pending message to acknowledge. " - << "Weird. grouter id = " << id << std::endl; + RsErr() << __PRETTY_FUNCTION__ + << " cannot find pending message to acknowledge. " + << "Weird. grouter id: " << id << std::endl; return; } uint32_t msg_id = it->second; - std::cerr << " message id = " << msg_id << std::endl; + + RsWarn() << __PRETTY_FUNCTION__ << " Global router tells " + << "us that item ID " << id + << " could not be delivered on time. Message id: " + << msg_id << std::endl; /* this is needed because it's not saved in config, but we should * probably include it in _ongoing_messages */ @@ -1912,10 +2032,10 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id, std::map::iterator mit = msgOutgoing.find(msg_id); if(mit == msgOutgoing.end()) { - std::cerr << " (II) message has been notified as not delivered, " - << "but it's not in outgoing list. Probably it has been " - << "delivered successfully by other means." - << std::endl; + RsInfo() << __PRETTY_FUNCTION__ + << " message has been notified as not delivered, " + << "but it's not in outgoing list. Probably it has been " + << "delivered successfully by other means." << std::endl; } else { @@ -1925,6 +2045,7 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id, // clear the routed flag so that the message is requested again mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED; } + return; } @@ -1954,17 +2075,31 @@ void p3MsgService::notifyDataStatus( const GRouterMsgPropagationId& id, return; } +#if 0 delete it2->second; msgOutgoing.erase(it2); +#else + // Do not delete it move to sent folder instead! + it2->second->msgFlags &= ~RS_MSG_FLAGS_PENDING; + imsg[msg_id] = it2->second; + msgOutgoing.erase(it2); +#endif RsServer::notify()->notifyListChange( NOTIFY_LIST_MESSAGELIST, NOTIFY_TYPE_ADD ); IndicateConfigChanged(); + using Evt_t = RsMailStatusEvent; + std::shared_ptr pEvent(new Evt_t()); + pEvent->mChangedMsgIds.insert(std::to_string(msg_id)); + if(rsEvents) rsEvents->postEvent(pEvent); + return; } - std::cerr << "p3MsgService: unhandled data status info from global router" - << " for msg ID " << id << ": this is a bug." << std::endl; + + RsErr() << __PRETTY_FUNCTION__ + << " unhandled data status info from global router" + << " for msg ID " << id << ": this is a bug." << std::endl; } bool p3MsgService::acceptDataFromPeer(const RsGxsId& to_gxs_id) @@ -1997,8 +2132,8 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId, const RsGxsId& recipientId, const uint8_t* data, uint32_t dataSize ) { - std::cout << __PRETTY_FUNCTION__ << " " << authorId << ", " << recipientId - << ",, " << dataSize << std::endl; + Dbg2() << __PRETTY_FUNCTION__ << " " << authorId << ", " << recipientId + << ",, " << dataSize << std::endl; Sha1CheckSum hash = RsDirUtil::sha1sum(data, dataSize); @@ -2007,29 +2142,31 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId, if( mRecentlyReceivedMessageHashes.find(hash) != mRecentlyReceivedMessageHashes.end() ) { - std::cerr << __PRETTY_FUNCTION__ << " (II) receiving " + RsInfo() << __PRETTY_FUNCTION__ << " (II) receiving " << "message of hash " << hash << " more than once. " << "Probably it has arrived before by other means." << std::endl; return true; } - mRecentlyReceivedMessageHashes[hash] = time(NULL); + mRecentlyReceivedMessageHashes[hash] = + static_cast(time(nullptr)); } IndicateConfigChanged(); - RsItem *item = _serialiser->deserialise(const_cast(data), &dataSize); + RsItem *item = _serialiser->deserialise( + const_cast(data), &dataSize ); RsMsgItem *msg_item = dynamic_cast(item); if(msg_item) { - std::cerr << __PRETTY_FUNCTION__ << " Encrypted item correctly " - << "deserialised. Passing on to incoming list." - << std::endl; + Dbg3() << __PRETTY_FUNCTION__ << " Encrypted item correctly " + << "deserialised. Passing on to incoming list." + << std::endl; msg_item->msgFlags |= RS_MSG_FLAGS_DISTANT; /* we expect complete msgs - remove partial flag just in case - * someone has funny ideas */ + * someone has funny ideas */ msg_item->msgFlags &= ~RS_MSG_FLAGS_PARTIAL; // hack to pass on GXS id. @@ -2038,8 +2175,8 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId, } else { - std::cerr << __PRETTY_FUNCTION__ << " Item could not be " - << "deserialised. Format error??" << std::endl; + RsWarn() << __PRETTY_FUNCTION__ << " Item could not be " + << "deserialised. Format error?" << std::endl; return false; } @@ -2049,8 +2186,11 @@ bool p3MsgService::receiveGxsTransMail( const RsGxsId& authorId, bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId, GxsTransSendStatus status ) { - std::cout << __PRETTY_FUNCTION__ << " " << mailId << ", " - << static_cast(status) << std::endl; + Dbg2() << __PRETTY_FUNCTION__ << " " << mailId << ", " + << static_cast(status) << std::endl; + + using Evt_t = RsMailStatusEvent; + std::shared_ptr pEvent(new Evt_t()); if( status == GxsTransSendStatus::RECEIPT_RECEIVED ) { @@ -2062,11 +2202,10 @@ bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId, auto it = gxsOngoingMessages.find(mailId); if(it == gxsOngoingMessages.end()) { - std::cerr << __PRETTY_FUNCTION__<< " " - << mailId - << ", " << static_cast(status) - << " (EE) cannot find pending message to acknowledge!" - << std::endl; + RsErr() << __PRETTY_FUNCTION__<< " " << mailId << ", " + << static_cast(status) + << " cannot find pending message to acknowledge!" + << std::endl; return false; } @@ -2081,26 +2220,32 @@ bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId, auto it2 = msgOutgoing.find(msg_id); if(it2 == msgOutgoing.end()) { - std::cerr << __PRETTY_FUNCTION__ << " " << mailId - << ", " << static_cast(status) << " (II) " - << "received receipt for message that is not in " - << "outgoing list, probably it has been acknoweldged " - << "before by other means." << std::endl; - return true; + RsInfo() << __PRETTY_FUNCTION__ << " " << mailId + << ", " << static_cast(status) + << " received receipt for message that is not in " + << "outgoing list, probably it has been acknoweldged " + << "before by other means." << std::endl; + } + else + { +#if 0 + delete it2->second; + msgOutgoing.erase(it2); +#else + // Do not delete it move to sent folder instead! + it2->second->msgFlags &= ~RS_MSG_FLAGS_PENDING; + imsg[msg_id] = it2->second; + msgOutgoing.erase(it2); +#endif + pEvent->mChangedMsgIds.insert(std::to_string(msg_id)); } - - delete it2->second; - msgOutgoing.erase(it2); } RsServer::notify()->notifyListChange( NOTIFY_LIST_MESSAGELIST, NOTIFY_TYPE_ADD ); IndicateConfigChanged(); - - return true; } - - if( status >= GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE ) + else if( status >= GxsTransSendStatus::FAILED_RECEIPT_SIGNATURE ) { uint32_t msg_id; @@ -2132,17 +2277,22 @@ bool p3MsgService::notifyGxsTransSendStatus( RsGxsTransId mailId, std::cerr << " message has been notified as not delivered, " << "but it not on outgoing list." << std::endl; - return true; } - std::cerr << " reseting the ROUTED flag so that the message is " - << "requested again" << std::endl; + else + { + std::cerr << " reseting the ROUTED flag so that the message is " + << "requested again" << std::endl; + // clear the routed flag so that the message is requested again + mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED; - // clear the routed flag so that the message is requested again - mit->second->msgFlags &= ~RS_MSG_FLAGS_ROUTED; - return true; + pEvent->mChangedMsgIds.insert(std::to_string(msg_id)); + } } } + if(rsEvents && !pEvent->mChangedMsgIds.empty()) + rsEvents->postEvent(pEvent); + return true; } diff --git a/libretroshare/src/services/p3msgservice.h b/libretroshare/src/services/p3msgservice.h index d94df85ba..61f5261db 100644 --- a/libretroshare/src/services/p3msgservice.h +++ b/libretroshare/src/services/p3msgservice.h @@ -37,7 +37,7 @@ #include "services/p3service.h" #include "rsitems/rsmsgitems.h" #include "util/rsthreads.h" - +#include "util/rsdebug.h" #include "retroshare/rsgxsifacetypes.h" #include "grouter/p3grouter.h" @@ -59,6 +59,19 @@ public: virtual RsServiceInfo getServiceInfo(); + /// @see RsMsgs::sendMail + uint32_t sendMail(const RsGxsId from, + const std::string& subject, + const std::string& body, + const std::set& to = std::set(), + const std::set& cc = std::set(), + const std::set& bcc = std::set(), + const std::vector& attachments = std::vector(), + std::set& trackingIds = + RS_DEFAULT_STORAGE_PARAM(std::set), + std::string& errorMsg = + RS_DEFAULT_STORAGE_PARAM(std::string) ); + /* External Interface */ bool getMessageSummaries(std::list &msgList); bool getMessage(const std::string &mid, Rs::Msgs::MessageInfo &msg); @@ -72,6 +85,7 @@ public: // msgParentId == 0 --> remove bool setMsgParentId(uint32_t msgId, uint32_t msgParentId); + RS_DEPRECATED_FOR(sendMail) bool MessageSend(Rs::Msgs::MessageInfo &info); bool SystemMessage(const std::string &title, const std::string &message, uint32_t systemFlag); bool MessageToDraft(Rs::Msgs::MessageInfo &info, const std::string &msgParentId); @@ -94,7 +108,6 @@ public: //std::list &getMsgOutList(); int tick(); - int status(); /*** Overloaded from p3Config ****/ virtual RsSerialiser *setupSerialiser(); @@ -227,6 +240,8 @@ private: bool mShouldEnableDistantMessaging ; p3GxsTrans& mGxsTransServ; + + RS_SET_CONTEXT_DEBUG_LEVEL(3) }; #endif // MESSAGE_SERVICE_HEADER diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index f1e96a8a4..328b04772 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -865,6 +865,8 @@ int p3turtle::handleIncoming() // void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) { + Dbg3() << __PRETTY_FUNCTION__ << " " << *item << std::endl; + // take a look at the item and test against inconsistent values // - If the item destimation is @@ -877,11 +879,12 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) if(item_size > TURTLE_MAX_SEARCH_REQ_ACCEPTED_SERIAL_SIZE) { -#ifdef P3TURTLE_DEBUG - std::cerr << " Dropping, because the serial size exceeds the accepted limit." << std::endl ; -#endif - std::cerr << " Caught a turtle search item with arbitrary large size from " << item->PeerId() << " of size " << item_size << " and depth " << item->depth << ". This is not allowed => dropping." << std::endl; - return ; + RsWarn() << __PRETTY_FUNCTION__ + << " Got a turtle search item with arbitrary large size from " + << item->PeerId() << " of size " << item_size << " and depth " + << item->depth << ". This is not allowed => dropping." + << std::endl; + return; } { @@ -889,22 +892,20 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE) { -#ifdef P3TURTLE_DEBUG - std::cerr << " Dropping, because the search request cache is full." << std::endl ; -#endif - std::cerr << " More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl; - return ; + RsWarn() << __PRETTY_FUNCTION__ << " More than " + << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. " + << "A peer is probably trying to flood your network See " + "the depth charts to find him." << std::endl; + return; } - // If the item contains an already handled search request, give up. This - // happens when the same search request gets relayed by different peers - // - if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end()) + if( _search_requests_origins.find(item->request_id) != + _search_requests_origins.end() ) { -#ifdef P3TURTLE_DEBUG - std::cerr << " This is a bouncing request. Ignoring and deleting it." << std::endl ; -#endif - return ; + /* If the item contains an already handled search request, give up. + * This happens when the same search request gets relayed by + * different peers */ + return; } } @@ -1013,13 +1014,21 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) // This function should be removed in the future, when file search will also use generic search items. -void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list& search_results,uint32_t& max_allowed_hits) +void p3turtle::performLocalSearch( + RsTurtleSearchRequestItem *item, uint32_t& req_result_count, + std::list& search_results, + uint32_t& max_allowed_hits ) { - RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast(item) ; + Dbg3() << __PRETTY_FUNCTION__ << " " << item << std::endl; + + RsTurtleFileSearchRequestItem* ftsearch = + dynamic_cast(item); if(ftsearch != NULL) { - performLocalSearch_files(ftsearch,req_result_count,search_results,max_allowed_hits) ; + performLocalSearch_files( + ftsearch, req_result_count, search_results, + max_allowed_hits ); return ; } @@ -1060,12 +1069,13 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item } } -void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list& result,uint32_t& max_allowed_hits) +void p3turtle::performLocalSearch_files( + RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count, + std::list& result, + uint32_t& max_allowed_hits ) { -#ifdef P3TURTLE_DEBUG - std::cerr << "Performing rsFiles->search()" << std::endl ; -#endif - // now, search! + Dbg3() << __PRETTY_FUNCTION__ << " " << *item << std::endl; + std::list initialResults ; item->search(initialResults) ; @@ -1104,6 +1114,9 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint res_item = NULL ; // forces creation of a new item. } } + + Dbg3() << __PRETTY_FUNCTION__ << " found " << req_result_count << " results" + << std::endl; } void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index a80025c28..6c0b0012d 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -19,6 +19,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ +#pragma once //====================================== General setup of the router ===================================// // @@ -130,10 +131,6 @@ // - should tunnels be re-used ? nope. The only useful case would be when two peers are exchanging files, which happens quite rarely. // - -#ifndef MRK_PQI_TURTLE_H -#define MRK_PQI_TURTLE_H - #include #include #include @@ -464,6 +461,8 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config uint32_t _service_type ; + RS_SET_CONTEXT_DEBUG_LEVEL(1) + #ifdef P3TURTLE_DEBUG // debug function void dumpState() ; @@ -472,5 +471,3 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config void TS_dumpState(); #endif }; - -#endif diff --git a/libretroshare/src/use_libretroshare.pri b/libretroshare/src/use_libretroshare.pri index 9a74d209c..f129f7f6c 100644 --- a/libretroshare/src/use_libretroshare.pri +++ b/libretroshare/src/use_libretroshare.pri @@ -68,11 +68,23 @@ linux-* { mLibs += dl } -rs_deep_search { +rs_deep_channels_index | rs_deep_files_index { mLibs += xapian win32-g++:mLibs += rpcrt4 } +rs_deep_files_index_ogg { + mLibs += vorbisfile +} + +rs_deep_files_index_flac { + mLibs += FLAC++ +} + +rs_deep_files_index_taglib { + mLibs += tag +} + rs_broadcast_discovery { no_rs_cross_compiling { UDP_DISCOVERY_SRC_PATH=$$clean_path($${RS_SRC_PATH}/supportlibs/udp-discovery-cpp/) diff --git a/libretroshare/src/util/rsurl.cc b/libretroshare/src/util/rsurl.cc index e625815e4..dd01d7384 100644 --- a/libretroshare/src/util/rsurl.cc +++ b/libretroshare/src/util/rsurl.cc @@ -1,6 +1,6 @@ /* * RetroShare - * Copyright (C) 2018 Gioacchino Mazzurco + * Copyright (C) 2018-2019 Gioacchino Mazzurco * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as @@ -17,15 +17,16 @@ */ -#include "rsurl.h" -#include "serialiser/rstypeserializer.h" - #include #include #include #include #include +#include "rsurl.h" +#include "serialiser/rstypeserializer.h" +#include "util/rsnet.h" + using namespace std; RsUrl::RsUrl() : mPort(0), mHasPort(false) {} @@ -33,6 +34,26 @@ RsUrl::RsUrl() : mPort(0), mHasPort(false) {} RsUrl::RsUrl(const std::string& urlStr) : mPort(0), mHasPort(false) { fromString(urlStr); } +RsUrl::RsUrl(const sockaddr_storage& addr): mPort(0), mHasPort(false) +{ + switch(addr.ss_family) + { + case AF_INET: setScheme("ipv4"); break; + case AF_INET6: setScheme("ipv6"); break; + default: + { + std::string addrDump; + sockaddr_storage_dump(addr, &addrDump); + RsErr() << __PRETTY_FUNCTION__ << " got invalid addr: " << addrDump + << std::endl; + return; + } + } + + setHost(sockaddr_storage_iptostring(addr)); + setPort(sockaddr_storage_port(addr)); +} + RsUrl& RsUrl::fromString(const std::string& urlStr) { size_t endI = urlStr.size()-1; @@ -55,7 +76,7 @@ RsUrl& RsUrl::fromString(const std::string& urlStr) { if(++hostBeginI >= endI) return *this; hostEndI = urlStr.find(ipv6WrapClose, hostBeginI); - mHost = urlStr.substr(hostBeginI, hostEndI - hostBeginI - 1); + mHost = urlStr.substr(hostBeginI, hostEndI - hostBeginI); ++hostEndI; } else @@ -250,10 +271,10 @@ RsUrl& RsUrl::setFragment(const std::string& fragment) size_t boundary = len-2; // % Encoded char must be at least 2 hex char for (size_t i = 0; i < len; ++i) { - if(str[i] == '%' && i < boundary) { - decoded << static_cast(std::stoi(str.substr(++i, 2), 0, 16)); + decoded << static_cast(std::stoi( + str.substr(++i, 2), nullptr, 16 )); ++i; } else decoded << str[i]; diff --git a/libretroshare/src/util/rsurl.h b/libretroshare/src/util/rsurl.h index f02c7f636..0610585c5 100644 --- a/libretroshare/src/util/rsurl.h +++ b/libretroshare/src/util/rsurl.h @@ -22,6 +22,8 @@ #include "serialiser/rsserializable.h" +struct sockaddr_storage; + /** * Very simplistic and minimal URL helper class for RetroShare, after looking * for a small and self-contained C/C++ URL parsing and manipulation library, @@ -36,7 +38,8 @@ struct RsUrl : RsSerializable { RsUrl(); - RsUrl(const std::string& urlStr); + explicit RsUrl(const std::string& urlStr); + explicit RsUrl(const sockaddr_storage& ssas); RsUrl& fromString(const std::string& urlStr); std::string toString() const; diff --git a/retroshare-gui/src/gui/ChatLobbyWidget.cpp b/retroshare-gui/src/gui/ChatLobbyWidget.cpp index a5e5579a0..dea3ea4e2 100644 --- a/retroshare-gui/src/gui/ChatLobbyWidget.cpp +++ b/retroshare-gui/src/gui/ChatLobbyWidget.cpp @@ -69,8 +69,8 @@ #define TYPE_LOBBY 1 #define IMAGE_CREATE "" -#define IMAGE_PUBLIC ":/images/chat_x24.png" -#define IMAGE_PRIVATE ":/images/chat_red24.png" +#define IMAGE_PUBLIC ":/icons/png/chats.png" +#define IMAGE_PRIVATE ":/icons/png/chats-private.png" #define IMAGE_SUBSCRIBE ":/images/edit_add24.png" #define IMAGE_UNSUBSCRIBE ":/images/cancel.png" #define IMAGE_PEER_ENTERING ":images/user/add_user24.png" @@ -211,6 +211,8 @@ ChatLobbyWidget::ChatLobbyWidget(QWidget *parent, Qt::WindowFlags flags) ).arg(QString::number(2*S)).arg(QString::number(S)) ; registerHelpButton(ui.helpButton,help_str,"ChatLobbyDialog") ; + + ui.lobbyTreeWidget->setIconSize(QSize(S*1.5,S*1.5)); } ChatLobbyWidget::~ChatLobbyWidget() diff --git a/retroshare-gui/src/gui/ChatLobbyWidget.ui b/retroshare-gui/src/gui/ChatLobbyWidget.ui index 134b50b9a..9a563f4a7 100644 --- a/retroshare-gui/src/gui/ChatLobbyWidget.ui +++ b/retroshare-gui/src/gui/ChatLobbyWidget.ui @@ -228,12 +228,18 @@ + + + - Selected Chat Room info + Chat Room info + + 6 + diff --git a/retroshare-gui/src/gui/chat/ChatWidget.cpp b/retroshare-gui/src/gui/chat/ChatWidget.cpp index cca487b59..94fca3d99 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.cpp +++ b/retroshare-gui/src/gui/chat/ChatWidget.cpp @@ -113,6 +113,7 @@ ChatWidget::ChatWidget(QWidget *parent) ui->searchButton->setIconSize(iconSize); ui->sendButton->setFixedHeight(iconHeight); ui->sendButton->setIconSize(iconSize); + ui->typingLabel->setMaximumHeight(QFontMetricsF(font()).height()*1.2); //Initialize search iCharToStartSearch=Settings->getChatSearchCharToStartSearch(); diff --git a/retroshare-gui/src/gui/chat/ChatWidget.ui b/retroshare-gui/src/gui/chat/ChatWidget.ui index 34ee50e53..a7af7c79f 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.ui +++ b/retroshare-gui/src/gui/chat/ChatWidget.ui @@ -7,16 +7,25 @@ 0 0 667 - 334 + 528 + + 0 + + + 0 + + + 0 + + + 0 + 2 - - 0 - @@ -37,7 +46,16 @@ QFrame::Raised - + + 0 + + + 0 + + + 0 + + 0 @@ -113,7 +131,16 @@ QFrame::Box - + + 6 + + + 6 + + + 6 + + 6 @@ -238,12 +265,6 @@ border-image: url(:/images/closepressed.png) - - - 400 - 18 - - 0 @@ -291,7 +312,7 @@ border-image: url(:/images/closepressed.png) 30 - + Type a message here @@ -311,7 +332,16 @@ border-image: url(:/images/closepressed.png) QFrame::Sunken - + + 2 + + + 2 + + + 2 + + 2 @@ -520,7 +550,7 @@ border-image: url(:/images/closepressed.png) Qt::NoFocus - + :/images/arrow-left.png:/images/arrow-left.png @@ -546,7 +576,7 @@ border-image: url(:/images/closepressed.png) Qt::NoFocus - + :/images/arrow-right.png:/images/arrow-right.png @@ -583,7 +613,16 @@ border-image: url(:/images/closepressed.png) QFrame::Plain - + + 2 + + + 2 + + + 2 + + 2 @@ -675,7 +714,16 @@ border-image: url(:/images/closepressed.png) QFrame::Sunken - + + 2 + + + 2 + + + 2 + + 2 @@ -756,7 +804,16 @@ border-image: url(:/images/closepressed.png) QFrame::Plain - + + 2 + + + 2 + + + 2 + + 2 @@ -998,6 +1055,7 @@ border-image: url(:/images/closepressed.png) + diff --git a/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp b/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp index 6f0ddd711..b93c2d438 100644 --- a/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp +++ b/retroshare-gui/src/gui/chat/PopupDistantChatDialog.cpp @@ -111,7 +111,7 @@ void PopupDistantChatDialog::updateDisplay() switch(tinfo.status) { case RS_DISTANT_CHAT_STATUS_UNKNOWN: - //std::cerr << "Unknown hash. Error!" << std::endl; + _status_label->setIcon(QIcon(IMAGE_GRY_LED)); msg = tr("Remote status unknown."); _status_label->setToolTip(msg); @@ -124,31 +124,31 @@ void PopupDistantChatDialog::updateDisplay() case RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED: std::cerr << "Chat remotely closed. " << std::endl; _status_label->setIcon(QIcon(IMAGE_RED_LED)); - _status_label->setToolTip( - QObject::tr("Distant peer has closed the chat") ); - getChatWidget()->updateStatusString( - "%1", tr( "The person you are talking to has deleted the" - " secured chat tunnel." ), true ); - getChatWidget()->blockSending(tr( "The chat partner deleted the secure" - " tunnel, messages will be delivered" - " as soon as possible")); + _status_label->setToolTip( QObject::tr("Distant peer has closed the chat") ); + + getChatWidget()->updateStatusString("%1", tr( "Your partner closed the conversation." ), true ); + getChatWidget()->blockSending(tr( "Your partner closed the conversation.")); + setPeerStatus(RS_STATUS_OFFLINE) ; break ; + case RS_DISTANT_CHAT_STATUS_TUNNEL_DN: - //std::cerr << "Tunnel asked. Waiting for reponse. " << std::endl; + _status_label->setIcon(QIcon(IMAGE_YEL_LED)); - msg = QObject::tr( "Tunnel is pending... Messages will be delivered as" - " soon as possible" ); + msg = QObject::tr( "Tunnel is pending"); + + if(tinfo.pending_items > 0) + msg += QObject::tr("(some undelivered messages)") ; // we cannot use the pending_items count because it accounts for ACKS and keep alive packets as well. + _status_label->setToolTip(msg); getChatWidget()->updateStatusString("%1", msg, true); getChatWidget()->blockSending(msg); setPeerStatus(RS_STATUS_OFFLINE); break; case RS_DISTANT_CHAT_STATUS_CAN_TALK: - //std::cerr << "Tunnel is ok and data is transmitted." << std::endl; + _status_label->setIcon(QIcon(IMAGE_GRN_LED)); - msg = QObject::tr( "Secured tunnel is working. " - "Messages are delivered immediately!" ); + msg = QObject::tr( "End-to-end encrypted conversation established"); _status_label->setToolTip(msg); getChatWidget()->unblockSending(); setPeerStatus(RS_STATUS_ONLINE); @@ -158,17 +158,15 @@ void PopupDistantChatDialog::updateDisplay() void PopupDistantChatDialog::closeEvent(QCloseEvent *e) { - //std::cerr << "Closing window => closing distant chat for hash " << _pid << std::endl; - DistantChatPeerInfo tinfo ; rsMsgs->getDistantChatStatus(_tunnel_id,tinfo) ; if(tinfo.status != RS_DISTANT_CHAT_STATUS_REMOTELY_CLOSED) { - QString msg = tr("Closing this window will end the conversation, notify the peer and remove the encrypted tunnel.") ; + QString msg = tr("Closing this window will end the conversation. Unsent messages will be dropped.") ; - if(QMessageBox::Ok == QMessageBox::critical(NULL,tr("Kill the tunnel?"),msg, QMessageBox::Ok | QMessageBox::Cancel)) + if(QMessageBox::Ok == QMessageBox::critical(NULL,tr("Close conversation?"),msg, QMessageBox::Ok | QMessageBox::Cancel)) rsMsgs->closeDistantChatConnexion(_tunnel_id) ; else { diff --git a/retroshare-gui/src/gui/icons.qrc b/retroshare-gui/src/gui/icons.qrc index 3445b1a41..abbdf8995 100644 --- a/retroshare-gui/src/gui/icons.qrc +++ b/retroshare-gui/src/gui/icons.qrc @@ -271,6 +271,7 @@ icons/png/correct.png icons/png/comment.png icons/png/chats.png + icons/png/chats-private.png icons/png/fileshare.png icons/png/forum.png icons/png/message.png diff --git a/retroshare-gui/src/gui/icons/png/chats-private.png b/retroshare-gui/src/gui/icons/png/chats-private.png new file mode 100644 index 000000000..039cc2b2d Binary files /dev/null and b/retroshare-gui/src/gui/icons/png/chats-private.png differ diff --git a/retroshare-gui/src/gui/icons/svg/chats-private.svg b/retroshare-gui/src/gui/icons/svg/chats-private.svg new file mode 100644 index 000000000..3086b0146 --- /dev/null +++ b/retroshare-gui/src/gui/icons/svg/chats-private.svg @@ -0,0 +1,64 @@ + + + +image/svg+xml \ No newline at end of file diff --git a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss index 05670f863..b817dd9cc 100644 --- a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss +++ b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss @@ -176,8 +176,9 @@ ChatLobbyWidget QGroupBox#lobbyinfo_groupBox ChatLobbyWidget QGroupBox::title#lobbyinfo_groupBox { - padding: 4 12px; - background: transparent; + padding: 4 12px; + background: #039bd5; + color: white; } ChatLobbyWidget QLabel#lobbyInfoLabel { diff --git a/retroshare-service/src/android/AndroidManifest.xml b/retroshare-service/src/android/AndroidManifest.xml index cbf2676fa..d41a72995 100644 --- a/retroshare-service/src/android/AndroidManifest.xml +++ b/retroshare-service/src/android/AndroidManifest.xml @@ -85,4 +85,8 @@ + + + diff --git a/retroshare.pri b/retroshare.pri index dcb9dc4cb..5224ecb81 100644 --- a/retroshare.pri +++ b/retroshare.pri @@ -159,16 +159,37 @@ rs_macos10.10:CONFIG -= rs_macos10.11 rs_macos10.12:CONFIG -= rs_macos10.11 rs_macos10.13:CONFIG -= rs_macos10.11 rs_macos10.14:CONFIG -= rs_macos10.11 +rs_macos10.15:CONFIG -= rs_macos10.11 # To enable JSON API append the following assignation to qmake command line # "CONFIG+=rs_jsonapi" CONFIG *= no_rs_jsonapi rs_jsonapi:CONFIG -= no_rs_jsonapi -# To enable deep search append the following assignation to qmake command line -# CONFIG *= rs_deep_search -CONFIG *= no_rs_deep_search -rs_deep_search:CONFIG -= no_rs_deep_search +# To enable channel indexing append the following assignation to qmake command +# line "CONFIG+=rs_deep_channel_index" +CONFIG *= no_rs_deep_channel_index +rs_deep_channel_index:CONFIG -= no_rs_deep_channel_index + +# To enable deep files indexing append the following assignation to qmake +# command line "CONFIG+=rs_files_index" +CONFIG *= no_rs_deep_files_index +rs_deep_files_index:CONFIG -= no_rs_deep_files_index + +# To enable Ogg files deep indexing append the following assignation to qmake +# command line "CONFIG+=rs_deep_files_index_ogg" +CONFIG *= no_rs_deep_files_index_ogg +rs_deep_files_index_ogg:CONFIG -= no_rs_deep_files_index_ogg + +# To enable FLAC files deep indexing append the following assignation to qmake +# command line "CONFIG+=rs_deep_files_index_flac" +CONFIG *= no_rs_deep_files_index_flac +rs_deep_files_index_flac:CONFIG -= no_rs_deep_files_index_flac + +# To enable taglib files deep indexing append the following assignation to qmake +# command line "CONFIG+=rs_deep_files_index_taglib" +CONFIG *= no_rs_deep_files_index_taglib +rs_deep_files_index_taglib:CONFIG -= no_rs_deep_files_index_taglib # To enable native dialogs append the following assignation to qmake command # line "CONFIG+=rs_use_native_dialogs" @@ -564,15 +585,12 @@ retroshare_qml_app { warning("QMAKE: you have enabled retroshare_qml_app which is deprecated") } -rs_deep_search { - DEFINES *= RS_DEEP_SEARCH +rs_deep_channels_index:DEFINES *= RS_DEEP_CHANNEL_INDEX - linux { - exists("/usr/include/xapian-1.3") { - INCLUDEPATH += /usr/include/xapian-1.3 - } - } -} +rs_deep_files_index:DEFINES *= RS_DEEP_FILES_INDEX +rs_deep_files_index_ogg:DEFINES *= RS_DEEP_FILES_INDEX_OGG +rs_deep_files_index_flac:DEFINES *= RS_DEEP_FILES_INDEX_FLAC +rs_deep_files_index_taglib:DEFINES *= RS_DEEP_FILES_INDEX_TAGLIB rs_use_native_dialogs:DEFINES *= RS_NATIVEDIALOGS @@ -743,6 +761,13 @@ macx-* { QMAKE_CXXFLAGS += -Wno-nullability-completeness QMAKE_CFLAGS += -Wno-nullability-completeness } + rs_macos10.15 { + message(***retroshare.pri: Set Target and SDK to MacOS 10.15 ) + QMAKE_MACOSX_DEPLOYMENT_TARGET=10.15 + QMAKE_MAC_SDK = macosx10.15 + QMAKE_CXXFLAGS += -Wno-nullability-completeness + QMAKE_CFLAGS += -Wno-nullability-completeness + }