Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-01-26 07:16:11 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Solve race condition in JSON API restart

Browse Source 
Improve atomic access to restbed service
Protect JSON API restart against bursts
Fix JSON API error condition enum registration
Provide ostream helper for error_condition
Provide optional forced thread cancel for debugging purpose, disabled by
  default at compile time define RS_THREAD_FORCE_STOP to enable it
Avoid double fullstop in retroshare-gui json api apply button
This commit is contained in:
Gioacchino Mazzurco 2020-02-05 15:01:45 +01:00
parent 7757c685c5
commit 039e8f653d
No known key found for this signature in database
GPG Key ID: A1FBCA3872E87051
8 changed files with 236 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
libretroshare/src/jsonapi/jsonapi.cpp
View File

@ -16,7 +16,7 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>
*
* SPDX-FileCopyrightText: 2004-2019 RetroShare Team <contact@retroshare.cc>
* SPDX-FileCopyrightText: 2004-2020 RetroShare Team <contact@retroshare.cc>
* SPDX-License-Identifier: AGPL-3.0-only
*/
@ -73,7 +73,7 @@ JsonApiServer::corsOptionsHeaders =
{ "Content-Length", "0" }
};
/* static */const RsJsonApiErrorCategory JsonApiServer::sErrorCategory;
/* static */ const RsJsonApiErrorCategory RsJsonApiErrorCategory::instance;
#define INITIALIZE_API_CALL_JSON_CONTEXT \
RsGenericSerializer::SerializeContext cReq( \
@ -126,6 +126,15 @@ JsonApiServer::corsOptionsHeaders =
return false;
}
void JsonApiServer::unProtectedRestart()
{
/* Extremely sensitive stuff!
* Make sure you read documentation in header before changing or use!! */
fullstop();
RsThread::start("JSON API Server");
}
bool RsJsonApi::parseToken(
const std::string& clear_token, std::string& user,std::string& passwd )
{
@ -143,13 +152,19 @@ bool RsJsonApi::parseToken(
return true;
}
JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
mService(std::make_shared<restbed::Service>()),
mServiceMutex("JsonApiServer restbed ptr"),
mService(nullptr),
mListeningPort(RsJsonApi::DEFAULT_PORT),
mBindingAddress(RsJsonApi::DEFAULT_BINDING_ADDRESS)
mBindingAddress(RsJsonApi::DEFAULT_BINDING_ADDRESS),
mRestartReqTS(0)
{
#if defined(RS_THREAD_FORCE_STOP) && defined(RS_JSONAPI_DEBUG_SERVICE_STOP)
/* When called in bursts it seems that Restbed::Service::stop() doesn't
* always does the job, to debug those cases it has been useful to ask
* RsThread to force it to stop for us. */
RsThread::setStopTimeout(10);
#endif
registerHandler("/rsLoginHelper/createLocation",
[this](const std::shared_ptr<rb::Session> session)
{
@ -394,6 +409,41 @@ JsonApiServer::JsonApiServer(): configMutex("JsonApiServer config"),
session->yield(message.str());
} );
}, true);
registerHandler("/rsJsonApi/restart",
[this](const std::shared_ptr<rb::Session> session)
{
auto reqSize = session->get_request()->get_header("Content-Length", 0);
session->fetch( static_cast<size_t>(reqSize), [this](
const std::shared_ptr<rb::Session> session,
const rb::Bytes& body )
{
INITIALIZE_API_CALL_JSON_CONTEXT;
std::error_condition retval;
const auto now = time(nullptr);
if(mRestartReqTS.exchange(now) + RESTART_BURST_PROTECTION > now)
retval = RsJsonApiErrorNum::NOT_A_MACHINE_GUN;
// serialize out parameters and return value to JSON
{
RsGenericSerializer::SerializeContext& ctx(cAns);
RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON);
RS_SERIAL_PROCESS(retval);
}
DEFAULT_API_CALL_JSON_RETURN(rb::OK);
/* Wrap inside RsThread::async because this call fullstop() on
* JSON API server thread.
* Calling RsThread::fullstop() from it's own thread should never
* happen and if it happens an error message is printed
* accordingly by RsThread::fullstop() */
if(!retval) RsThread::async([this](){ unProtectedRestart(); });
} );
}, true);
// Generated at compile time
#include "jsonapi-wrappers.inl"
}
@ -464,9 +514,7 @@ void JsonApiServer::registerHandler(
std::string tUser;
parseToken(authToken, tUser, RS_DEFAULT_STORAGE_PARAM(std::string));
authFail(rb::UNAUTHORIZED)
<< " user: " << tUser
<< " error: " << ec.value() << " " << ec.message()
<< std::endl;
<< " user: " << tUser << ec << std::endl;
}
} );
@ -661,21 +709,21 @@ std::vector<std::shared_ptr<rb::Resource> > JsonApiServer::getResources() const
return tab;
}
void JsonApiServer::restart()
std::error_condition JsonApiServer::restart()
{
/* It is important to wrap into async(...) because fullstop() method can't
* be called from same thread of execution hence from JSON API thread! */
RsThread::async([this]()
{
fullstop();
RsThread::start("JSON API Server");
});
const auto now = time(nullptr);
if(mRestartReqTS.exchange(now) + RESTART_BURST_PROTECTION > now)
return RsJsonApiErrorNum::NOT_A_MACHINE_GUN;
unProtectedRestart();
return std::error_condition();
}
void JsonApiServer::onStopRequested()
{
RS_STACK_MUTEX(mServiceMutex);
mService->stop();
auto tService = std::atomic_exchange(
&mService, std::shared_ptr<rb::Service>(nullptr) );
if(tService) tService->stop();
}
uint16_t JsonApiServer::listeningPort() const { return mListeningPort; }
@ -691,14 +739,9 @@ void JsonApiServer::run()
settings->set_bind_address(mBindingAddress);
settings->set_default_header("Connection", "close");
/* re-allocating mService is important because it deletes the existing
* service and therefore leaves the listening port open */
{
RS_STACK_MUTEX(mServiceMutex);
mService = std::make_shared<restbed::Service>();
}
auto tService = std::make_shared<restbed::Service>();
for(auto& r: getResources()) mService->publish(r);
for(auto& r: getResources()) tService->publish(r);
try
{
@ -706,7 +749,20 @@ void JsonApiServer::run()
.setPort(mListeningPort);
RsInfo() << __PRETTY_FUNCTION__ << " JSON API server listening on "
<< apiUrl.toString() << std::endl;
mService->start(settings);
/* re-allocating mService is important because it deletes the existing
* service and therefore leaves the listening port open */
auto tExpected = std::shared_ptr<rb::Service>(nullptr);
if(atomic_compare_exchange_strong(&mService, &tExpected, tService))
tService->start(settings);
else
{
RsErr() << __PRETTY_FUNCTION__ << " mService was expected to be "
<< " null, instead we got: " << tExpected
<< " something wrong happened JsonApiServer won't start"
<< std::endl;
print_stacktrace();
}
}
catch(std::exception& e)
{
@ -730,11 +786,6 @@ void JsonApiServer::run()
human = RS_HUMAN_READABLE_VERSION;
}
std::error_condition make_error_condition(RsJsonApiErrorNum e)
{
return std::error_condition(static_cast<int>(e), rsJsonApi->errorCategory());
}
std::error_condition RsJsonApiErrorCategory::default_error_condition(int ev) const noexcept
{
switch(static_cast<RsJsonApiErrorNum>(ev))

36
libretroshare/src/jsonapi/jsonapi.h
View File

@ -30,6 +30,7 @@
#include <set>
#include <functional>
#include <vector>
#include <atomic>
#include "util/rsthreads.h"
#include "pqi/p3cfgmgr.h"
@ -40,6 +41,7 @@
namespace rb = restbed;
/** Interface to provide addotional resources to JsonApiServer */
class JsonApiResourceProvider
{
public:
@ -66,7 +68,7 @@ public:
void fullstop() override { RsThread::fullstop(); }
/// @see RsJsonApi
void restart() override;
std::error_condition restart() override;
/// @see RsJsonApi
void askForStop() override { RsThread::askForStop(); }
@ -145,10 +147,6 @@ public:
const std::function<bool(const std::string&, const std::string&)>&
callback );
/// @see RsJsonApi
const std::error_category& errorCategory() override
{ return sErrorCategory; }
protected:
/// @see RsThread
void onStopRequested() override;
@ -206,13 +204,33 @@ private:
std::reference_wrapper<const JsonApiResourceProvider>,
std::less<const JsonApiResourceProvider> > mResourceProviders;
/**
* This pointer should be accessed via std::atomic_* operations, up until
* now only very critical operations like reallocation, are done that way,
* but this is not still 100% thread safe, but seems to handle all of the
* test cases (no crash, no deadlock), once we switch to C++20 we shoud
* change this into std::atomic<std::shared_ptr<restbed::Service>> which
* will automatically handle atomic access properly all the times
*/
std::shared_ptr<restbed::Service> mService;
/** Protect service only during very critical operation like resetting the
* pointer, still not 100% thread safe, but hopefully we can avoid
* crashes/freeze with this */
RsMutex mServiceMutex;
uint16_t mListeningPort;
std::string mBindingAddress;
/// @see unProtectedRestart()
std::atomic<rstime_t> mRestartReqTS;
/// @see unProtectedRestart()
constexpr static rstime_t RESTART_BURST_PROTECTION = 7;
/** It is very important to protect this method from being called in bursts,
* because Restbed::Service::stop() together with
* Restbed::Service::start(...), which are called internally, silently fails
* if combined in bursts, probably because they have to deal with
* listening/releasing TCP port.
* @see JsonApiServer::restart() and @see JsonApiServer::JsonApiServer()
* implementation to understand how correctly use this.
*/
void unProtectedRestart();
};

40
libretroshare/src/retroshare/rsjsonapi.h
View File

@ -1,8 +1,9 @@
/*
* RetroShare JSON API public header
*
* Copyright (C) 2018-2019 Gioacchino Mazzurco <gio.eigenlab.org>
* Copyright (C) 2018-2020 Gioacchino Mazzurco <gio.eigenlab.org>
* Copyright (C) 2019 Cyril Soler <csoler@users.sourceforge.net>
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net>
*
* This program is free software: you can redistribute it and/or modify it under
* the terms of the GNU Lesser General Public License as published by the Free
@ -45,7 +46,8 @@ enum class RsJsonApiErrorNum : int32_t
WRONG_API_PASSWORD = 2006,
API_USER_CONTAIN_COLON = 2007,
AUTHORIZATION_REQUEST_DENIED = 2008,
CANNOT_EXECUTE_BEFORE_RS_LOGIN = 2009
CANNOT_EXECUTE_BEFORE_RS_LOGIN = 2009,
NOT_A_MACHINE_GUN = 2010
};
struct RsJsonApiErrorCategory: std::error_category
@ -69,21 +71,36 @@ struct RsJsonApiErrorCategory: std::error_category
return "User denied new token autorization";
case RsJsonApiErrorNum::CANNOT_EXECUTE_BEFORE_RS_LOGIN:
return "This operation cannot be executed bedore RetroShare login";
case RsJsonApiErrorNum::NOT_A_MACHINE_GUN:
return "Method must not be called in burst";
default:
return "Error message for error:" + std::to_string(ev) +
" not available";
return "Error message for error: " + std::to_string(ev) +
" not available in category: " + name();
}
}
std::error_condition default_error_condition(int ev) const noexcept override;
const static RsJsonApiErrorCategory instance;
};
namespace std
{
template<> struct is_error_condition_enum<RsJsonApiErrorNum>: true_type {};
error_condition make_error_condition(RsJsonApiErrorNum e);
/** Register RsJsonApiErrorNum as an error condition enum, must be in std
* namespace */
template<> struct is_error_condition_enum<RsJsonApiErrorNum> : true_type {};
}
/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in
* same namespace of RsJsonApiErrorNum */
inline std::error_condition make_error_condition(RsJsonApiErrorNum e) noexcept
{
return std::error_condition(
static_cast<int>(e), RsJsonApiErrorCategory::instance );
};
class p3ConfigMgr;
class JsonApiResourceProvider;
@ -94,10 +111,12 @@ public:
static const std::string DEFAULT_BINDING_ADDRESS; // 127.0.0.1
/**
* @brief Restart RsJsonApi server asynchronously.
* @jsonapi{development}
* @brief Restart RsJsonApi server.
* This method is asyncronous when called from JSON API.
* @jsonapi{development,manualwrapper}
* @return Success or error details
*/
virtual void restart() = 0;
virtual std::error_condition restart() = 0;
/** @brief Request RsJsonApi to stop and wait until it has stopped.
* Do not expose this method to JSON API as fullstop must not be called from
@ -234,9 +253,6 @@ public:
static void version( uint32_t& major, uint32_t& minor, uint32_t& mini,
std::string& extra, std::string& human );
/** @brief Return a reference to service error category */
virtual const std::error_category& errorCategory() = 0;
virtual ~RsJsonApi() = default;
};

27
libretroshare/src/util/rsdebug.cc
View File

@ -3,7 +3,9 @@
* *
* libretroshare: retroshare core library *
* *
* Copyright 2004-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2004-2008 by Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -21,11 +23,25 @@
*******************************************************************************/
#include "util/rsdebug.h"
#include "util/rsthreads.h"
#include "util/rsdir.h"
std::ostream &operator<<(std::ostream& out, const std::error_condition& err)
{
return out << " error: " << err.value() << " " << err.message()
<< " category: " << err.category().name();
}
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
////////////////////////////////////////////////////////////////////////////////
/// All the following lines are DEPRECATED!!
#include <map>
#include <stdio.h>
#include <cstdio>
#include "util/rsthreads.h"
#include "util/rsdir.h"
#include "util/rstime.h"
const int RS_DEBUG_STDERR = 1; /* stuff goes to stderr */
@ -186,6 +202,3 @@ void rslog(const RsLog::logLvl lvl, RsLog::logInfo *info, const std::string &msg
lineCount++;
}
}

9
libretroshare/src/util/rsdebug.h
View File

@ -2,7 +2,8 @@
* RetroShare debugging utilities *
* *
* Copyright (C) 2004-2008 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2020 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
@ -21,6 +22,10 @@
#pragma once
#include <ostream>
#include <system_error>
/** Stream helper for std::error_condition */
std::ostream &operator<<(std::ostream& out, const std::error_condition& err);
#ifdef __ANDROID__
# include <android/log.h>
@ -101,7 +106,7 @@ struct t_RsLogger
/**
* Comfortable debug message loggin, supports chaining like std::cerr but can
* be easly and selectively disabled at compile time to reduce generated binary
* size and performance impact without too many #ifdef around.
* size and performance impact without too many \#ifdef around.
*
* To selectively debug your context you can just add something like this in
* in that context, as an example for a class you can just add a line like this

37
libretroshare/src/util/rsthreads.cc
View File

@ -4,7 +4,7 @@
* libretroshare: retroshare core library *
* *
* Copyright (C) 2004-2007 Robert Fernie <retroshare@lunamutt.com> *
* Copyright (C) 2016-2019 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2016-2020 Gioacchino Mazzurco <gio@eigenlab.org> *
* Copyright (C) 2019-2020 Asociación Civil Altermundi <info@altermundi.net> *
* *
* This program is free software: you can redistribute it and/or modify *
@ -23,7 +23,7 @@
*******************************************************************************/
#include <iostream>
#include <time.h>
#include <ctime>
#include <thread>
#include <chrono>
@ -95,6 +95,9 @@ void RsThread::resetTid()
}
RsThread::RsThread() : mHasStopped(true), mShouldStop(false), mLastTid()
#ifdef RS_THREAD_FORCE_STOP
, mStopTimeout(0)
#endif
{ resetTid(); }
bool RsThread::isRunning() { return !mHasStopped; }
@ -117,6 +120,10 @@ void RsThread::wrapRun()
void RsThread::fullstop()
{
#ifdef RS_THREAD_FORCE_STOP
const rstime_t stopRequTS = time(nullptr);
#endif
askForStop();
const pthread_t callerTid = pthread_self();
@ -141,6 +148,32 @@ void RsThread::fullstop()
RsDbg() << __PRETTY_FUNCTION__ << " " << i*0.2 << " seconds passed"
<< " waiting for thread: " << std::hex << mLastTid
<< std::dec << " " << mFullName << " to stop" << std::endl;
#ifdef RS_THREAD_FORCE_STOP
if(mStopTimeout && time(nullptr) > stopRequTS + mStopTimeout)
{
RsErr() << __PRETTY_FUNCTION__ << " thread mLastTid: " << std::hex
<< mLastTid << " mTid: " << mTid << std::dec << " "
<< mFullName
<< " ignored our nice stop request for more then "
<< mStopTimeout
<< " seconds, will be forcefully stopped. "
<< "Please submit a report to RetroShare developers"
<< std::endl;
const auto terr = pthread_cancel(mTid);
if(terr == 0) mHasStopped = true;
else
{
RsErr() << __PRETTY_FUNCTION__ << " pthread_cancel("
<< std::hex << mTid << std::dec <<") returned "
<< terr << " " << rsErrnoName(terr) << std::endl;
print_stacktrace();
}
break;
}
#endif // def RS_THREAD_FORCE_STOP
}
}

15
libretroshare/src/util/rsthreads.h
View File

@ -35,6 +35,9 @@
#include "util/rsmemory.h"
#include "util/rsdeprecate.h"
#ifdef RS_THREAD_FORCE_STOP
# include "util/rstime.h"
#endif
//#define RSMUTEX_DEBUG
@ -249,6 +252,13 @@ protected:
* of this method, @see JsonApiServer for an usage example. */
virtual void onStopRequested() {}
#ifdef RS_THREAD_FORCE_STOP
/** Set last resort timeout to forcefully kill thread if it didn't stop
* nicely, one should never use this, still we needed to introduce this
* to investigate some bugs in external libraries */
void setStopTimeout(rstime_t timeout) { mStopTimeout = timeout; }
#endif
private:
/** Call @see run() setting the appropriate flags around it*/
void wrapRun();
@ -277,6 +287,11 @@ private:
* and that might happens concurrently (or just before) a debug message
* being printed, thus causing the debug message to print a mangled value.*/
pthread_t mLastTid;
#ifdef RS_THREAD_FORCE_STOP
/// @see setStopTimeout
rstime_t mStopTimeout;
#endif
};
/**

29
retroshare-gui/src/gui/settings/JsonApiPage.cc
View File

@ -112,28 +112,41 @@ QString JsonApiPage::helpText() const { return ""; }
bool JsonApiPage::checkStartJsonApi()
{
if(!Settings->getJsonApiEnabled())
return false;
if(!Settings->getJsonApiEnabled()) return false;
rsJsonApi->setListeningPort(Settings->getJsonApiPort());
rsJsonApi->setBindingAddress(Settings->getJsonApiListenAddress().toStdString());
rsJsonApi->restart();
const auto rErr = rsJsonApi->restart();
if(rErr == RsJsonApiErrorNum::NOT_A_MACHINE_GUN)
{
RsDbg() << __PRETTY_FUNCTION__ << " apparently the user is attempting "
<< "to restart JSON API service in a burst. Re-scheduling "
<< "restart in a while..." << std::endl;
RsThread::async([]()
{
std::this_thread::sleep_for(std::chrono::seconds(10));
rsJsonApi->restart();
});
}
else if(rErr)
{
RsErr() << __PRETTY_FUNCTION__ << rErr << std::endl;
return false;
}
return true;
}
/*static*/ void JsonApiPage::checkShutdownJsonApi()
{
if(!rsJsonApi->isRunning()) return;
rsJsonApi->fullstop(); // this is a blocks until the thread is terminated.
}
void JsonApiPage::onApplyClicked()
{
// restart
checkShutdownJsonApi();
checkStartJsonApi();
// restart
checkStartJsonApi();
}
void JsonApiPage::checkToken(QString s)