Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-05-03 22:55:04 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'upstream/master' into pqihandlerOptim

Browse source
This commit is contained in:
jolavillette 2020-06-04 07:39:59 +02:00
commit 752c997f24
57 changed files with 500 additions and 881 deletions
Download patch file Download diff file Expand all files Collapse all files

10
libretroshare/src/file_sharing/p3filelists.cc
View file

@ -1424,9 +1424,15 @@ int p3FileDatabase::SearchBoolExp(RsRegularExpression::Expression *exp, std::lis
return !results.empty() ;
}
bool p3FileDatabase::search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const
bool p3FileDatabase::search(
const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const
{
RS_STACK_MUTEX(mFLSMtx) ;
RS_STACK_MUTEX(mFLSMtx);
if( (hintflags & RS_FILE_HINTS_EXTRA) &&
mExtraFiles->search(hash, hintflags, info) )
return true;
if(hintflags & RS_FILE_HINTS_LOCAL)
{

65
libretroshare/src/ft/ftextralist.cc
View file

@ -4,7 +4,7 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -21,6 +21,9 @@
* *
*******************************************************************************/
#include <limits>
#include <system_error>
#ifdef WINDOWS_SYS
#include "util/rswin.h"
#endif
@ -245,12 +248,8 @@ bool ftExtraList::cleanupOldFiles()
/* remove items */
for(std::list<RsFileHash>::iterator rit = toRemove.begin(); rit != toRemove.end(); ++rit)
{
if (mFiles.end() != (it = mFiles.find(*rit)))
{
cleanupEntry(it->second.info.path, it->second.info.transfer_info_flags);
mFiles.erase(it);
}
mHashOfHash.erase(makeEncryptedHash(*rit)) ;
if (mFiles.end() != (it = mFiles.find(*rit))) mFiles.erase(it);
mHashOfHash.erase(makeEncryptedHash(*rit));
}
IndicateConfigChanged();
@ -258,46 +257,39 @@ bool ftExtraList::cleanupOldFiles()
return true;
}
bool ftExtraList::cleanupEntry(std::string /*path*/, TransferRequestFlags /*flags*/)
{
// if (flags & RS_FILE_CONFIG_CLEANUP_DELETE)
// {
// /* Delete the file? - not yet! */
// }
return true;
}
/***
* Hash file, and add to the files,
* file is removed after period.
**/
bool ftExtraList::hashExtraFile(
std::string path, uint32_t period, TransferRequestFlags flags )
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::hashExtraFile() path: " << path;
std::cerr << " period: " << period;
std::cerr << " flags: " << flags;
constexpr rstime_t max_int = std::numeric_limits<int>::max();
const rstime_t now = time(nullptr);
const rstime_t timeOut = now + period;
std::cerr << std::endl;
#endif
auto failure = [](std::string errMsg)
if(timeOut > max_int)
{
RsErr() << __PRETTY_FUNCTION__ << " " << errMsg << std::endl;
/* Under the hood period is stored as int FileInfo::age so we do this
* check here to detect 2038 year problem
* https://en.wikipedia.org/wiki/Year_2038_problem */
RsErr() << __PRETTY_FUNCTION__ << " period: " << period << " > "
<< max_int - now << std::errc::value_too_large << std::endl;
return false;
};
}
if(!RsDirUtil::fileExists(path))
return failure("file: " + path + "not found");
{
RsErr() << __PRETTY_FUNCTION__ << " path: " << path
<< std::errc::no_such_file_or_directory << std::endl;
return false;
}
if(RsDirUtil::checkDirectory(path))
return failure("Cannot add a directory: " + path + "as extra file");
{
RsErr() << __PRETTY_FUNCTION__ << " path: " << path
<< std::errc::is_a_directory << std::endl;
return false;
}
FileDetails details(path, period, flags);
details.info.age = static_cast<int>(time(nullptr) + period);
details.info.age = static_cast<int>(timeOut);
{
RS_STACK_MUTEX(extMutex);
@ -492,8 +484,7 @@ bool ftExtraList::loadList(std::list<RsItem *>& load)
if (ts > (rstime_t)fi->file.age)
{
/* to old */
cleanupEntry(fi->file.path, TransferRequestFlags(fi->flags));
/* too old */
delete (*it);
continue ;
}

9
libretroshare/src/ft/ftextralist.h
View file

@ -60,7 +60,7 @@
#include "pqi/p3cfgmgr.h"
#include "util/rstime.h"
class FileDetails
class RS_DEPRECATED_FOR(FileInfo) FileDetails
{
public:
FileDetails()
@ -130,7 +130,11 @@ public:
* file is removed after period.
**/
bool hashExtraFile(std::string path, uint32_t period, TransferRequestFlags flags);
/**
* Hash file, and add to the files, file is removed after period.
*/
bool hashExtraFile(
std::string path, uint32_t period, TransferRequestFlags flags );
bool hashExtraFileDone(std::string path, FileInfo &info);
/***
@ -165,7 +169,6 @@ private:
/* Worker Functions */
void hashAFile();
bool cleanupOldFiles();
bool cleanupEntry(std::string path, TransferRequestFlags flags);
mutable RsMutex extMutex;

13
libretroshare/src/ft/ftserver.cc
View file

@ -22,6 +22,8 @@
#include <algorithm>
#include <iostream>
#include <limits>
#include <system_error>
#include "crypto/chacha20.h"
//const int ftserverzone = 29539;
@ -293,7 +295,8 @@ bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& re
bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
{
return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info);
return mFileDatabase->search(
hash, RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_LOCAL, info );
}
bool ftServer::FileRequest(
@ -819,6 +822,14 @@ bool ftServer::ExtraFileRemove(const RsFileHash& hash)
bool ftServer::ExtraFileHash(
std::string localpath, rstime_t period, TransferRequestFlags flags )
{
constexpr rstime_t uintmax = std::numeric_limits<uint32_t>::max();
if(period > uintmax)
{
RsErr() << __PRETTY_FUNCTION__ << " period: " << period << " > "
<< uintmax << std::errc::value_too_large << std::endl;
return false;
}
return mFtExtra->hashExtraFile(
localpath, static_cast<uint32_t>(period), flags );
}

2
libretroshare/src/libretroshare.pro
View file

@ -870,7 +870,7 @@ rs_jsonapi {
genrestbedheader.name = Generating restbed header.
genrestbedheader.input = genrestbedlib.output
genrestbedheader.output = $${RESTBED_HEADER_FILE}
genrestbedheader.CONFIG += target_predeps combine no_link
genrestbedheader.CONFIG += target_predeps no_link
genrestbedheader.variable_out = HEADERS
genrestbedheader.commands = cd $${RESTBED_BUILD_PATH} && $(MAKE) install
QMAKE_EXTRA_COMPILERS += genrestbedheader

3
libretroshare/src/retroshare/rsfiles.h
View file

@ -658,7 +658,8 @@ public:
* @brief Get file details
* @jsonapi{development}
* @param[in] hash file identifier
* @param[in] hintflags filtering hint (RS_FILE_HINTS_EXTRA|...|RS_FILE_HINTS_LOCAL)
* @param[in] hintflags filtering hint ( RS_FILE_HINTS_UPLOAD|...|
* RS_FILE_HINTS_EXTRA|RS_FILE_HINTS_LOCAL )
* @param[out] info storage for file information
* @return true if file found, false otherwise
*/

1
libretroshare/src/retroshare/rsgxschannels.h
View file

@ -310,6 +310,7 @@ public:
* @param[in] channelId id of the channel of which the content is requested
* @param[out] posts storage for posts
* @param[out] comments storage for the comments
* @param[out] votes storage for votes
* @return false if something failed, true otherwhise
*/
virtual bool getChannelAllContent( const RsGxsGroupId& channelId,

89
libretroshare/src/serialiser/rstypeserializer.cc
View file

@ -503,8 +503,16 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/,
// Binary blocks //
//============================================================================//
#if __cplusplus < 201703L
/* Solve weird undefined reference error with C++ < 17 see:
* https://stackoverflow.com/questions/8016780/undefined-reference-to-static-constexpr-char
*/
/*static*/ decltype(RsTypeSerializer::RawMemoryWrapper::base64_key) constexpr
RsTypeSerializer::RawMemoryWrapper::base64_key;
/*static*/ /* without this Android compilation breaks */
constexpr uint32_t RsTypeSerializer::RawMemoryWrapper::MAX_SERIALIZED_CHUNK_SIZE;
#endif
/*static*/
void RsTypeSerializer::RawMemoryWrapper::serial_process(
@ -542,18 +550,7 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process(
ctx.mOffset += second;
break;
case RsGenericSerializer::DESERIALIZE:
if(first || second)
{
/* Items are created anew before deserialization so buffer pointer
* must be null and size 0 at this point */
RsWarn() << __PRETTY_FUNCTION__ << " DESERIALIZE got uninitialized "
<< " or pre-allocated buffer! Buffer pointer: " << first
<< " must be null and size: " << second << " must be 0 at "
<< "this point. Does your item costructor initialize them "
<< "properly?" << std::endl;
print_stacktrace();
}
freshMemCheck();
RS_SERIAL_PROCESS(second);
if(!ctx.mOk) break;
@ -597,44 +594,33 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process(
if(!ctx.mOk) break;
std::string encodedValue;
RsBase64::encode(first, second, encodedValue, true, false);
ctx.mJson.SetString(
encodedValue.data(),
static_cast<rapidjson::SizeType>(encodedValue.length()),
ctx.mJson.GetAllocator());
ctx.mOk = ctx.mOk &&
RsTypeSerializer::to_JSON(base64_key, encodedValue, ctx.mJson);
break;
}
case RsGenericSerializer::FROM_JSON:
{
const bool yelding = !!(
RsSerializationFlags::YIELDING & ctx.mFlags );
if(!(ctx.mOk || yelding))
{
clear();
break;
}
if(!ctx.mJson.IsString())
{
RsErr() << __PRETTY_FUNCTION__ << " "
<< std::errc::invalid_argument << std::endl;
print_stacktrace();
freshMemCheck();
ctx.mOk = false;
clear();
break;
}
if( ctx.mJson.GetStringLength() >
const auto failure = [&]() -> void { ctx.mOk = false; clear(); };
const bool yielding = !!(
RsSerializationFlags::YIELDING & ctx.mFlags );
if(!(ctx.mOk || yielding)) return failure();
std::string encodedValue;
if(!RsTypeSerializer::from_JSON(
base64_key, encodedValue, ctx.mJson )) return failure();
if( encodedValue.length() >
RsBase64::encodedSize(MAX_SERIALIZED_CHUNK_SIZE, true) )
{
RsErr() << __PRETTY_FUNCTION__ << " "
<< std::errc::message_size << std::endl;
print_stacktrace();
ctx.mOk = false;
clear();
break;
return failure();
}
std::string encodedValue = ctx.mJson.GetString();
std::vector<uint8_t> decoded;
auto ec = RsBase64::decode(encodedValue, decoded);
if(ec)
@ -642,9 +628,7 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process(
RsErr() << __PRETTY_FUNCTION__ << " " << ec << std::endl;
print_stacktrace();
ctx.mOk = false;
clear();
break;
return failure();
}
const auto decodedSize = decoded.size();
@ -655,11 +639,8 @@ void RsTypeSerializer::RawMemoryWrapper::serial_process(
break;
}
if(decodedSize != second)
{
first = reinterpret_cast<uint8_t*>(realloc(first, decodedSize));
second = static_cast<uint32_t>(decodedSize);
}
first = reinterpret_cast<uint8_t*>(malloc(decodedSize));
second = static_cast<uint32_t>(decodedSize);
memcpy(first, decoded.data(), second);
break;
@ -675,6 +656,24 @@ void RsTypeSerializer::RawMemoryWrapper::clear()
second = 0;
}
bool RsTypeSerializer::RawMemoryWrapper::freshMemCheck()
{
if(first || second)
{
/* Items are created anew before deserialization so buffer pointer
* must be null and size 0 at this point */
RsWarn() << __PRETTY_FUNCTION__ << " got uninitialized "
<< " or pre-allocated buffer! Buffer pointer: " << first
<< " must be null and size: " << second << " must be 0 at "
<< "this point. Does your item costructor initialize them "
<< "properly?" << std::endl;
print_stacktrace();
return false;
}
return true;
}
//============================================================================//
// std::error_condition //
//============================================================================//

17
libretroshare/src/serialiser/rstypeserializer.h
View file

@ -59,12 +59,17 @@ struct RsTypeSerializer
/// Maximum supported size 10MB
static constexpr uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024;
/** Key used for JSON serialization.
* @note Changing this value breaks JSON API retro-compatibility */
static constexpr char base64_key[] = "base64";
/// @see RsSerializable
void serial_process(
RsGenericSerializer::SerializeJob j,
RsGenericSerializer::SerializeContext& ctx ) override;
private:
void clear();
bool freshMemCheck();
};
/// Most types are not valid sequence containers
@ -777,9 +782,9 @@ struct RsTypeSerializer
{
if(!yielding)
{
std::cerr << __PRETTY_FUNCTION__ << " \"" << memberName
<< "\" not found in JSON:" << std::endl
<< jDoc << std::endl << std::endl;
RsErr() << __PRETTY_FUNCTION__ << " \"" << memberName
<< "\" not found in JSON:" << std::endl
<< jDoc << std::endl << std::endl;
print_stacktrace();
}
ctx.mOk = false;
@ -790,9 +795,9 @@ struct RsTypeSerializer
if(!v.IsObject())
{
std::cerr << __PRETTY_FUNCTION__ << " \"" << memberName
<< "\" has wrong type in JSON, object expected, got:"
<< std::endl << jDoc << std::endl << std::endl;
RsErr() << __PRETTY_FUNCTION__ << " \"" << memberName
<< "\" has wrong type in JSON, object expected, got:"
<< std::endl << jDoc << std::endl << std::endl;
print_stacktrace();
ctx.mOk = false;
break;

6
libretroshare/src/services/p3msgservice.cc
View file

@ -1187,11 +1187,15 @@ bool p3MsgService::MessageSend(MessageInfo &info)
/* use processMsg to get the new msgId */
msg->recvTime = time(NULL);
msg->msgId = getNewUniqueMsgId();
msg->msgFlags |= RS_MSG_OUTGOING;
imsg[msg->msgId] = msg;
// Update info for caller
info.msgId = std::to_string(msg->msgId);
info .msgflags = msg->msgFlags;
RsServer::notify()->notifyListChange(NOTIFY_LIST_MESSAGELIST,NOTIFY_TYPE_ADD);
}

9
libretroshare/src/services/p3photoservice.cc
View file

@ -117,11 +117,7 @@ void p3PhotoService::groupsChanged(std::list<RsGxsGroupId>& grpIds)
while(!mGroupChange.empty())
{
RsGxsGroupChange* gc = mGroupChange.back();
std::list<RsGxsGroupId>& gList = gc->mGrpIdList;
std::list<RsGxsGroupId>::iterator lit = gList.begin();
for(; lit != gList.end(); ++lit) {
grpIds.push_back(*lit);
}
grpIds.push_back(gc->mGroupId);
mGroupChange.pop_back();
delete gc;
@ -136,7 +132,8 @@ void p3PhotoService::msgsChanged(GxsMsgIdResult& msgs)
while(!mMsgChange.empty())
{
RsGxsMsgChange* mc = mMsgChange.back();
msgs = mc->msgChangeMap;
msgs[mc->mGroupId].insert(mc->mMsgId);
mMsgChange.pop_back();
delete mc;
}

16
libretroshare/src/util/rsbase64.cc
View file

@ -21,8 +21,6 @@
* *
*******************************************************************************/
#include <cmath>
#include "util/rsbase64.h"
#include "util/rsdebug.h"
@ -40,6 +38,12 @@
rs_view_ptr<const uint8_t> data, size_t len, std::string& outString,
bool padding, bool urlSafe )
{
if(!data || !len)
{
outString.clear();
return;
}
const char* sDict = urlSafe ? uDict : bDict;
// Workaround if input and output are the same buffer.
@ -137,9 +141,11 @@
/*static*/ size_t RsBase64::encodedSize(size_t decodedSize, bool padding)
{
if(padding) return 4 * (decodedSize + 2) / 3;
return static_cast<size_t>(
std::ceil(4L * static_cast<double>(decodedSize) / 3L) );
if(!decodedSize) return 0;
// Thanks https://stackoverflow.com/a/45401395
if(padding) return ceilDivision(decodedSize, 3) * 4;
return ceilDivision(decodedSize * 8, 6);
}
/*static*/ std::tuple<size_t, std::error_condition> RsBase64::decodedSize(

4
libretroshare/src/util/rsbase64.h
View file

@ -137,4 +137,8 @@ private:
*/
static inline bool isBase64Char(char c)
{ return rDict[static_cast<uint8_t>(c)] >= 0; }
/** Perform ceil division without floating point operations */
static inline size_t ceilDivision(size_t dividend, size_t divisor)
{ return (dividend + divisor - 1) / divisor; }
};

79
libretroshare/src/util/rsdebug.h
View file

@ -32,6 +32,8 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
# include <sstream>
# include <string>
# include "util/rsjson.h"
enum class RsLoggerCategories
{
DEBUG = ANDROID_LOG_DEBUG,
@ -46,6 +48,18 @@ struct t_RsLogger
{
inline t_RsLogger() = default;
/** Offer variadic style too, as a benefit this has better atomicity then
* << style, but doesn't supports manipulators and things like std::endl
* @see https://stackoverflow.com/a/27375675 */
template <typename Arg, typename... Args>
inline t_RsLogger(Arg&& arg, Args&&... args)
{
ostr << std::forward<Arg>(arg);
using expander = int[];
(void)expander{0, (void(ostr << std::forward<Args>(args)), 0)...};
mFlush();
}
/** On other platforms expose the type of underlying stream.
* On Android it cannot work like that so return the class type itself
* just for code compatibility with other platforms */
@ -55,17 +69,16 @@ struct t_RsLogger
inline stream_type& operator<<(const T& val)
{ ostr << val; return *this; }
template<typename T>
inline stream_type& operator<<(const RsJson& val)
{ ostr << val; return *this; }
/// needed for manipulators and things like std::endl
stream_type& operator<<(std::ostream& (*pf)(std::ostream&))
{
if(pf == static_cast<std::ostream& (*)(std::ostream&)>(
&std::endl< char, std::char_traits<char> > ))
{
__android_log_write(
static_cast<int>(CATEGORY),
"RetroShare", ostr.str().c_str() );
ostr.str() = "";
}
mFlush();
else ostr << pf;
return *this;
@ -78,6 +91,14 @@ struct t_RsLogger
private:
std::ostringstream ostr;
void mFlush()
{
__android_log_write(
static_cast<int>(CATEGORY),
"RetroShare", ostr.str().c_str() );
ostr.str() = "";
}
};
#else // def __ANDROID__
@ -99,38 +120,56 @@ enum class RsLoggerCategories
template <RsLoggerCategories CATEGORY>
struct t_RsLogger
{
inline t_RsLogger() = default;
/// Expose the type of underlying stream
using stream_type = decltype(std::cerr);
/// Return underlying stream to write avoiding additional prefixes
static inline stream_type& uStream() { return std::cerr; }
inline t_RsLogger() = default;
/** Offer variadic style too, as a benefit this has better atomicity then
* << style, but doesn't supports manipulators and things like std::endl
* @see https://stackoverflow.com/a/27375675 */
template <typename Arg, typename... Args>
inline t_RsLogger(Arg&& arg, Args&&... args)
{
std::ostringstream ostr;
ostr << getPrefix() << std::forward<Arg>(arg);
using expander = int[];
(void)expander{0, (void(ostr << std::forward<Args>(args)), 0)...};
ostr << std::endl;
uStream() << ostr.str();
}
template<typename T>
inline stream_type& operator<<(const T& val)
{ return uStream() << getPrefix() << val; }
/// needed for manipulators and things like std::endl
stream_type& operator<<(std::ostream& (*pf)(std::ostream&))
{ return uStream() << pf; }
private:
std::string getPrefix()
{
using namespace std::chrono;
const auto now = system_clock::now();
const auto sec = time_point_cast<seconds>(now);
const auto msec = duration_cast<milliseconds>(now - sec);
std::stringstream tstream;
std::ostringstream tstream;
tstream << static_cast<char>(CATEGORY) << " "
<< sec.time_since_epoch().count() << "."
<< std::setfill('0') << std::setw(3) << msec.count()
<< " " << val;
return std::cerr << tstream.str();
<< " ";
return tstream.str();
}
/// needed for manipulators and things like std::endl
stream_type& operator<<(std::ostream& (*pf)(std::ostream&))
{ return std::cerr << pf; }
/// Return underlying stream to write avoiding additional prefixes
inline stream_type& uStream() const { return std::cerr; }
};
#endif // def __ANDROID__
/**
* Comfortable debug message loggin, supports chaining like std::cerr but can
* Comfortable debug message logging, supports chaining like std::cerr but can
* be easly and selectively disabled at compile time to reduce generated binary
* size and performance impact without too many \#ifdef around.
*
@ -190,6 +229,8 @@ struct RsNoDbg
{
inline RsNoDbg() = default;
template <typename T, typename... Args> inline RsNoDbg(T, Args...) {}
/** Defined as the type itself just for code compatibility with other
* logging classes */
using stream_type = RsNoDbg;

2
libretroshare/src/util/rsjson.h
View file

@ -29,7 +29,7 @@
typedef rapidjson::Document RsJson;
/**
* Print out RsJson to a stream, use std::stringstream to get the string
* Print out RsJson to a stream, use std::ostringstream to get the string
* @param[out] out output stream
* @param[in] jDoc JSON document to print
* @return same output stream passed as out parameter