From 4b6f751b09a76159295b1fac373564df1be01be1 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 16 Aug 2018 23:34:29 +0200 Subject: [PATCH] Implement JSON API generation for async API calls Move JSON helpers to util/rsjson.* for better usability Implement JSON ostream manipulator to print compact and pretty JSON Use lambdas for API wrappers, integrate better and avoid namespace pollution Removed experimental JSON API for notify client wrapper, notifications can be implemented automatically with moderns async API calls Implement and automatically expose to JSON API RsGxsChannels::turtleSearchRequest( const std::string& matchString, const std::function& multiCallback, std::time_t maxWait ) --- .../async-method-wrapper-template.cpp.tmpl | 73 ++++++++++ jsonapi-generator/src/jsonapi-generator.cpp | 137 ++++++++++++++---- .../src/method-wrapper-template.cpp.tmpl | 12 +- libretroshare/src/jsonapi/jsonapi.cpp | 79 ++-------- libretroshare/src/jsonapi/jsonapi.h | 16 -- libretroshare/src/libretroshare.pro | 33 ++--- libretroshare/src/retroshare/rsgxschannels.h | 15 ++ libretroshare/src/retroshare/rsgxsiface.h | 1 + libretroshare/src/serialiser/rsserializer.h | 8 +- .../src/serialiser/rstypeserializer.cc | 13 -- .../src/serialiser/rstypeserializer.h | 7 +- libretroshare/src/services/p3gxschannels.cc | 63 +++++++- libretroshare/src/services/p3gxschannels.h | 40 ++++- libretroshare/src/util/rsjson.cc | 68 +++++++++ libretroshare/src/util/rsjson.h | 56 +++++++ 15 files changed, 444 insertions(+), 177 deletions(-) create mode 100644 jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl create mode 100644 libretroshare/src/util/rsjson.cc create mode 100644 libretroshare/src/util/rsjson.h diff --git a/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl b/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl new file mode 100644 index 000000000..8dd03a505 --- /dev/null +++ b/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl @@ -0,0 +1,73 @@ +/* + * RetroShare JSON API + * Copyright (C) 2018 Gioacchino Mazzurco + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +registerHandler("$%apiPath%$", + [$%captureVars%$](const std::shared_ptr session) +{ + const std::multimap headers + { + { "Connection", "keep-alive" }, + { "Content-Type", "text/event-stream" } + }; + session->yield(rb::OK, headers); + + size_t reqSize = session->get_request()->get_header("Content-Length", 0); + session->fetch( reqSize, [$%captureVars%$]( + const std::shared_ptr session, + const rb::Bytes& body ) + { + RsGenericSerializer::SerializeContext cReq( + nullptr, 0, + RsGenericSerializer::SERIALIZATION_FLAG_YIELDING ); + RsJson& jReq(cReq.mJson); + jReq.Parse(reinterpret_cast(body.data()), body.size()); + + RsGenericSerializer::SerializeContext cAns; + RsJson& jAns(cAns.mJson); + + // if caller specified caller_data put it back in the answhere + const char kcd[] = "caller_data"; + if(jReq.HasMember(kcd)) + jAns.AddMember(kcd, jReq[kcd], jAns.GetAllocator()); + +$%paramsDeclaration%$ + +$%inputParamsDeserialization%$ + + $%callbackName%$ = [session]($%callbackParams%$) + { +$%callbackParamsSerialization%$ + + std::stringstream message; + message << "data: " << compactJSON << ctx.mJson << "\n\n"; + session->yield(message.str()); + $%sessionEarlyClose%$ + }; + +$%functionCall%$ + +$%outputParamsSerialization%$ + + // return them to the API caller + std::stringstream message; + message << "data: " << compactJSON << cAns.mJson << "\n\n"; + session->yield(message.str()); + $%sessionDelayedClose%$ + } ); +}); + diff --git a/jsonapi-generator/src/jsonapi-generator.cpp b/jsonapi-generator/src/jsonapi-generator.cpp index e0dd7659d..9d6a04bad 100644 --- a/jsonapi-generator/src/jsonapi-generator.cpp +++ b/jsonapi-generator/src/jsonapi-generator.cpp @@ -25,12 +25,15 @@ struct MethodParam { - MethodParam() : in(false), out(false) {} + MethodParam() : + in(false), out(false), isMultiCallback(false), isSingleCallback(false){} QString type; QString name; bool in; bool out; + bool isMultiCallback; + bool isSingleCallback; }; int main(int argc, char *argv[]) @@ -42,24 +45,18 @@ int main(int argc, char *argv[]) QString outputPath(argv[2]); QString doxPrefix(outputPath+"/xml/"); - QString wrappersDefFilePath(outputPath + "/jsonapi-wrappers.cpp"); + QString wrappersDefFilePath(outputPath + "/jsonapi-wrappers.inl"); QFile wrappersDefFile(wrappersDefFilePath); wrappersDefFile.remove(); if(!wrappersDefFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text)) qFatal(QString("Can't open: " + wrappersDefFilePath).toLatin1().data()); - QString wrappersDeclFilePath(outputPath + "/jsonapi-wrappers.h"); - QFile wrappersDeclFile(wrappersDeclFilePath); - wrappersDeclFile.remove(); - if(!wrappersDeclFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text)) - qFatal(QString("Can't open: " + wrappersDeclFilePath).toLatin1().data()); - - QString wrappersRegisterFilePath(outputPath + "/jsonapi-register.inl"); - QFile wrappersRegisterFile(wrappersRegisterFilePath); - wrappersRegisterFile.remove(); - if(!wrappersRegisterFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text)) - qFatal(QString("Can't open: " + wrappersRegisterFilePath).toLatin1().data()); - + QString cppApiIncludesFilePath(outputPath + "/jsonapi-includes.inl"); + QFile cppApiIncludesFile(cppApiIncludesFilePath); + cppApiIncludesFile.remove(); + if(!cppApiIncludesFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text)) + qFatal(QString("Can't open: " + cppApiIncludesFilePath).toLatin1().data()); + QSet cppApiIncludesSet; QDirIterator it(doxPrefix, QStringList() << "*8h.xml", QDir::Files); while(it.hasNext()) @@ -151,21 +148,45 @@ int main(int argc, char *argv[]) QString retvalType = memberdef.firstChildElement("type").text(); QMap paramsMap; QStringList orderedParamNames; - uint hasInput = false; - uint hasOutput = false; + bool hasInput = false; + bool hasOutput = false; + bool hasSingleCallback = false; + bool hasMultiCallback = false; + QString callbackName; + QString callbackParams; QDomNodeList params = memberdef.elementsByTagName("param"); for (int k = 0; k < params.size(); ++k) { QDomElement tmpPE = params.item(k).toElement(); MethodParam tmpParam; - tmpParam.name = tmpPE.firstChildElement("declname").text(); - QDomElement tmpType = tmpPE.firstChildElement("type"); + QString& pName(tmpParam.name); QString& pType(tmpParam.type); + pName = tmpPE.firstChildElement("declname").text(); + QDomElement tmpType = tmpPE.firstChildElement("type"); pType = tmpType.text(); if(pType.startsWith("const ")) pType.remove(0,6); - pType.replace(QString("&"), QString()); - pType.replace(QString(" "), QString()); + if(pType.startsWith("std::function")) + { + if(pType.endsWith('&')) pType.chop(1); + if(pName.startsWith("multiCallback")) + { + tmpParam.isMultiCallback = true; + hasMultiCallback = true; + } + else if(pName.startsWith("callback")) + { + tmpParam.isSingleCallback = true; + hasSingleCallback = true; + } + callbackName = pName; + callbackParams = pType; + } + else + { + pType.replace(QString("&"), QString()); + pType.replace(QString(" "), QString()); + } paramsMap.insert(tmpParam.name, tmpParam); orderedParamNames.push_back(tmpParam.name); } @@ -242,6 +263,58 @@ int main(int argc, char *argv[]) "\t\t\tRS_SERIAL_PROCESS(retval);\n"; if(hasOutput) outputParamsSerialization += "\t\t}\n"; + QString captureVars; + + QString sessionEarlyClose; + if(hasSingleCallback) + sessionEarlyClose = "session->close();"; + + QString sessionDelayedClose; + if(hasMultiCallback) + { + sessionDelayedClose = "mService.schedule( [session](){session->close();}, std::chrono::seconds(maxWait+120) );"; + captureVars = "this"; + } + + QString callbackParamsSerialization; + + if(hasSingleCallback || hasMultiCallback || + ((callbackParams.indexOf('(')+2) < callbackParams.indexOf(')'))) + { + QString& cbs(callbackParamsSerialization); + + callbackParams = callbackParams.split('(')[1]; + callbackParams = callbackParams.split(')')[0]; + + cbs += "RsGenericSerializer::SerializeContext ctx;\n"; + + for (QString cbPar : callbackParams.split(',')) + { + bool isConst(cbPar.startsWith("const ")); + QChar pSep(' '); + bool isRef(cbPar.contains('&')); + if(isRef) pSep = '&'; + int sepIndex = cbPar.lastIndexOf(pSep)+1; + QString cpt(cbPar.mid(0, sepIndex)); + cpt.remove(0,6); + QString cpn(cbPar.mid(sepIndex)); + + cbs += "\t\t\tRsTypeSerializer::serial_process("; + cbs += "RsGenericSerializer::TO_JSON, ctx, "; + if(isConst) + { + cbs += "const_cast<"; + cbs += cpt; + cbs += ">("; + } + cbs += cpn; + if(isConst) cbs += ")"; + cbs += ", \""; + cbs += cpn; + cbs += "\" );\n"; + } + } + QMap substitutionsMap; substitutionsMap.insert("paramsDeclaration", paramsDeclaration); substitutionsMap.insert("inputParamsDeserialization", inputParamsDeserialization); @@ -249,8 +322,20 @@ int main(int argc, char *argv[]) substitutionsMap.insert("wrapperName", wrapperName); substitutionsMap.insert("headerFileName", headerFileName); substitutionsMap.insert("functionCall", functionCall); + substitutionsMap.insert("apiPath", apiPath); + substitutionsMap.insert("sessionEarlyClose", sessionEarlyClose); + substitutionsMap.insert("sessionDelayedClose", sessionDelayedClose); + substitutionsMap.insert("captureVars", captureVars); + substitutionsMap.insert("callbackName", callbackName); + substitutionsMap.insert("callbackParams", callbackParams); + substitutionsMap.insert("callbackParamsSerialization", callbackParamsSerialization); - QFile templFile(sourcePath + "/method-wrapper-template.cpp.tmpl"); + QString templFilePath(sourcePath); + if(hasMultiCallback || hasSingleCallback) + templFilePath.append("/async-method-wrapper-template.cpp.tmpl"); + else templFilePath.append("/method-wrapper-template.cpp.tmpl"); + + QFile templFile(templFilePath); templFile.open(QIODevice::ReadOnly); QString wrapperDef(templFile.readAll()); @@ -261,15 +346,13 @@ int main(int argc, char *argv[]) wrappersDefFile.write(wrapperDef.toLocal8Bit()); - QString wrapperDecl("void " + instanceName + methodName + "Wrapper(const std::shared_ptr session);\n"); - wrappersDeclFile.write(wrapperDecl.toLocal8Bit()); - - - QString wrapperReg("registerHandler(\""+apiPath+"\", "+wrapperName+");\n"); - wrappersRegisterFile.write(wrapperReg.toLocal8Bit()); + cppApiIncludesSet.insert("#include \"retroshare/" + headerFileName + "\"\n"); } } } + for(const QString& incl : cppApiIncludesSet) + cppApiIncludesFile.write(incl.toLocal8Bit()); + return 0; } diff --git a/jsonapi-generator/src/method-wrapper-template.cpp.tmpl b/jsonapi-generator/src/method-wrapper-template.cpp.tmpl index 417001ec6..f9be77d76 100644 --- a/jsonapi-generator/src/method-wrapper-template.cpp.tmpl +++ b/jsonapi-generator/src/method-wrapper-template.cpp.tmpl @@ -16,14 +16,8 @@ * along with this program. If not, see . */ -#include -#include -#include -#include "retroshare/$%headerFileName%$" - -namespace rb = restbed; - -void $%wrapperName%$(const std::shared_ptr session) +registerHandler("$%apiPath%$", + [$%captureVars%$](const std::shared_ptr session) { size_t reqSize = session->get_request()->get_header("Content-Length", 0); session->fetch( reqSize, []( @@ -66,5 +60,5 @@ $%outputParamsSerialization%$ }; session->close(rb::OK, ans, headers); } ); -} +}); diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index ee4ef1158..3c2ee6cda 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -18,16 +18,19 @@ #include "jsonapi.h" -#include -#include -#include +#include +#include +#include + +#include "util/rsjson.h" +#include "retroshare/rsgxschannels.h" // Generated at compile time -#include "jsonapi-wrappers.h" +#include "jsonapi-includes.inl" JsonApiServer::JsonApiServer( uint16_t port, const std::function shutdownCallback) : - mPort(port), mShutdownCallback(shutdownCallback), notifyClientWrapper(*this) + mPort(port), mShutdownCallback(shutdownCallback) { registerHandler("/jsonApiServer/shutdown", [this](const std::shared_ptr) @@ -35,32 +38,16 @@ JsonApiServer::JsonApiServer( shutdown(); }); - registerHandler("/jsonApiServer/notifications", - [this](const std::shared_ptr session) - { - const auto headers = std::multimap - { - { "Connection", "keep-alive" }, - { "Cache-Control", "no-cache" }, - { "Content-Type", "text/event-stream" }, - }; - - session->yield(rb::OK, headers, - [this](const std::shared_ptr session) - { - notifySessions.push_back(session); - } ); - } ); - // Generated at compile time -#include "jsonapi-register.inl" +#include "jsonapi-wrappers.inl" } void JsonApiServer::run() { std::shared_ptr settings(new rb::Settings); settings->set_port(mPort); - settings->set_default_header("Connection", "close"); +// settings->set_default_header("Connection", "close"); + settings->set_default_header("Cache-Control", "no-cache"); mService.start(settings); } @@ -80,47 +67,3 @@ void JsonApiServer::shutdown(int exitCode) mService.stop(); mShutdownCallback(exitCode); } - -void JsonApiServer::cleanClosedNotifySessions() -{ - notifySessions.erase( - std::remove_if( - notifySessions.begin(), notifySessions.end(), - [](const std::shared_ptr &s) - { return s->is_closed(); } ), notifySessions.end()); -} - -JsonApiServer::NotifyClientWrapper::NotifyClientWrapper(JsonApiServer& parent) : - NotifyClient(), mJsonApiServer(parent) -{ - rsNotify->registerNotifyClient(static_cast(this)); -} - -void JsonApiServer::NotifyClientWrapper::notifyTurtleSearchResult( - uint32_t searchId, const std::list& files ) -{ - mJsonApiServer.cleanClosedNotifySessions(); - - RsGenericSerializer::SerializeContext cAns; - RsJson& jAns(cAns.mJson); - - // serialize parameters and method name to JSON - { - std::string methodName("NotifyClient/notifyTurtleSearchResult"); - std::list filesCopy(files); - RsGenericSerializer::SerializeContext& ctx(cAns); - RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON); - RS_SERIAL_PROCESS(methodName); - RS_SERIAL_PROCESS(searchId); -// RS_SERIAL_PROCESS(filesCopy); - } - - rapidjson::StringBuffer buffer; - rapidjson::Writer writer(buffer); - jAns.Accept(writer); - std::string message(buffer.GetString(), buffer.GetSize()); - message.insert(0, "data: "); message.append("\n\n"); - - for(auto session : mJsonApiServer.notifySessions) - session->yield(message); -} diff --git a/libretroshare/src/jsonapi/jsonapi.h b/libretroshare/src/jsonapi/jsonapi.h index bb3f78378..f06f7e459 100644 --- a/libretroshare/src/jsonapi/jsonapi.h +++ b/libretroshare/src/jsonapi/jsonapi.h @@ -21,7 +21,6 @@ #include #include "util/rsthreads.h" -#include "retroshare/rsnotify.h" namespace rb = restbed; @@ -70,20 +69,5 @@ private: uint16_t mPort; rb::Service mService; const std::function mShutdownCallback; - - std::list > notifySessions; - void cleanClosedNotifySessions(); - - struct NotifyClientWrapper : NotifyClient - { - NotifyClientWrapper(JsonApiServer& parent); - - void notifyTurtleSearchResult( - uint32_t searchId, const std::list& files); - - private: - JsonApiServer& mJsonApiServer; - }; - NotifyClientWrapper notifyClientWrapper; }; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 2e2d0feed..8b5c879db 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -776,10 +776,12 @@ SOURCES += gxstunnel/p3gxstunnel.cc \ # new serialization code HEADERS += serialiser/rsserializable.h \ serialiser/rsserializer.h \ - serialiser/rstypeserializer.h + serialiser/rstypeserializer.h \ + util/rsjson.h SOURCES += serialiser/rsserializer.cc \ - serialiser/rstypeserializer.cc + serialiser/rstypeserializer.cc \ + util/rsjson.cc # Identity Service HEADERS += retroshare/rsidentity.h \ @@ -874,9 +876,8 @@ rs_jsonapi { DOXIGEN_INPUT_DIRECTORY=$$system_path($$clean_path($${PWD})) DOXIGEN_CONFIG_SRC=$$system_path($$clean_path($${RS_SRC_PATH}/jsonapi-generator/src/jsonapi-generator-doxygen.conf)) DOXIGEN_CONFIG_OUT=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-generator-doxygen.conf)) - WRAPPERS_DEF_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-wrappers.cpp)) - WRAPPERS_DECL_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-wrappers.h)) - WRAPPERS_REG_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-register.inl)) + WRAPPERS_INCL_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-includes.inl)) + WRAPPERS_REG_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-wrappers.inl)) restbed.target = $$system_path($$clean_path($${RESTBED_BUILD_PATH}/library/librestbed.a)) restbed.commands = \ @@ -890,30 +891,22 @@ rs_jsonapi { PRE_TARGETDEPS *= $${JSONAPI_GENERATOR_EXE} INCLUDEPATH *= $${JSONAPI_GENERATOR_OUT} - GENERATED_HEADERS += $${WRAPPERS_DECL_FILE} $${WRAPPERS_REG_FILE} - GENERATED_SOURCES += $${WRAPPERS_DEF_FILE} + GENERATED_HEADERS += $${WRAPPERS_INCL_FILE} - jsonwrappersdecl.target = $${WRAPPERS_DECL_FILE} - jsonwrappersdecl.commands = \ + jsonwrappersincl.target = $${WRAPPERS_INCL_FILE} + jsonwrappersincl.commands = \ cp $${DOXIGEN_CONFIG_SRC} $${DOXIGEN_CONFIG_OUT}; \ echo OUTPUT_DIRECTORY=$${JSONAPI_GENERATOR_OUT} >> $${DOXIGEN_CONFIG_OUT};\ echo INPUT=$${DOXIGEN_INPUT_DIRECTORY} >> $${DOXIGEN_CONFIG_OUT}; \ doxygen $${DOXIGEN_CONFIG_OUT}; \ $${JSONAPI_GENERATOR_EXE} $${JSONAPI_GENERATOR_SRC} $${JSONAPI_GENERATOR_OUT}; - QMAKE_EXTRA_TARGETS += jsonwrappersdecl - libretroshare.depends += jsonwrappersdecl - PRE_TARGETDEPS *= $${WRAPPERS_DECL_FILE} - - jsonwrappersdef.target = $${WRAPPERS_DEF_FILE} - jsonwrappersdef.commands = touch $${WRAPPERS_DEF_FILE} - jsonwrappersdef.depends = jsonwrappersdecl - QMAKE_EXTRA_TARGETS += jsonwrappersdef - libretroshare.depends += jsonwrappersdef - PRE_TARGETDEPS *= $${WRAPPERS_DEF_FILE} + QMAKE_EXTRA_TARGETS += jsonwrappersincl + libretroshare.depends += jsonwrappersincl + PRE_TARGETDEPS *= $${WRAPPERS_INCL_FILE} jsonwrappersreg.target = $${WRAPPERS_REG_FILE} jsonwrappersreg.commands = touch $${WRAPPERS_REG_FILE} - jsonwrappersreg.depends = jsonwrappersdef + jsonwrappersreg.depends = jsonwrappersincl QMAKE_EXTRA_TARGETS += jsonwrappersreg libretroshare.depends += jsonwrappersreg PRE_TARGETDEPS *= $${WRAPPERS_REG_FILE} diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index f7d2430a6..754ea7a28 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -25,6 +25,7 @@ #include #include #include +#include #include "retroshare/rstokenservice.h" #include "retroshare/rsgxsifacehelper.h" @@ -266,6 +267,20 @@ public: */ virtual bool ExtraFileRemove(const RsFileHash& hash) = 0; + /** + * @brief Request remote channels search + * @jsonapi{development} + * @param[in] matchString string to look for in the search + * @param multiCallback function that will be called each time a search + * result is received + * @param[in] maxWait maximum wait time in seconds for search results + * @return false on error, true otherwise + */ + virtual bool turtleSearchRequest( + const std::string& matchString, + const std::function& multiCallback, + std::time_t maxWait = 300 ) = 0; + ////////////////////////////////////////////////////////////////////////////// /// Distant synchronisation methods /// ////////////////////////////////////////////////////////////////////////////// diff --git a/libretroshare/src/retroshare/rsgxsiface.h b/libretroshare/src/retroshare/rsgxsiface.h index 4adc3fce7..7fbf4b691 100644 --- a/libretroshare/src/retroshare/rsgxsiface.h +++ b/libretroshare/src/retroshare/rsgxsiface.h @@ -32,6 +32,7 @@ #include "gxs/rsgxsdata.h" #include "retroshare/rsgxsifacetypes.h" #include "util/rsdeprecate.h" +#include "serialiser/rsserializable.h" /*! * This structure is used to transport group summary information when a GXS diff --git a/libretroshare/src/serialiser/rsserializer.h b/libretroshare/src/serialiser/rsserializer.h index 357d075dc..09803eaa6 100644 --- a/libretroshare/src/serialiser/rsserializer.h +++ b/libretroshare/src/serialiser/rsserializer.h @@ -151,15 +151,11 @@ #include #include #include -#ifdef HAS_RAPIDJSON -#include -#else -#include -#endif // HAS_RAPIDJSON #include "retroshare/rsflags.h" #include "serialiser/rsserial.h" #include "util/rsdeprecate.h" +#include "util/rsjson.h" struct RsItem; @@ -198,8 +194,6 @@ class RsRawSerialiser: public RsSerialType virtual RsItem * deserialise(void *data, uint32_t *size); }; -typedef rapidjson::Document RsJson; - /// Top class for all services and config serializers. struct RsGenericSerializer : RsSerialType { diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index 69908e117..9c2ebc37b 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -660,16 +660,3 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/, RsTypeSerializer::TlvMemBlock_proxy&, RsJson& /*jDoc*/) { return true; } - - -//============================================================================// -// RsJson std:ostream support // -//============================================================================// - -std::ostream &operator<<(std::ostream &out, const RsJson &jDoc) -{ - rapidjson::StringBuffer buffer; buffer.Clear(); - rapidjson::PrettyWriter writer(buffer); - jDoc.Accept(writer); - return out << buffer.GetString(); -} diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index de488480c..d1ccd8520 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -31,12 +31,8 @@ #include "serialiser/rsserializer.h" #include "serialiser/rsserializable.h" +#include "util/rsjson.h" -#ifdef HAS_RAPIDJSON -#include -#else -#include -#endif // HAS_RAPIDJSON #include // for typeid #include #include @@ -116,7 +112,6 @@ \ } while(false) -std::ostream &operator<<(std::ostream &out, const RsJson &jDoc); struct RsTypeSerializer { diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index 26c892b71..386fe4ee0 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -69,7 +69,8 @@ p3GxsChannels::p3GxsChannels( RsGixs* gixs ) : RsGenExchange( gds, nes, new RsGxsChannelSerialiser(), RS_SERVICE_GXS_TYPE_CHANNELS, gixs, channelsAuthenPolicy() ), - RsGxsChannels(static_cast(*this)), GxsTokenQueue(this) + RsGxsChannels(static_cast(*this)), GxsTokenQueue(this), + mSearchCallbacksMapMutex("GXS channels search") { // For Dummy Msgs. mGenActive = false; @@ -351,8 +352,6 @@ static time_t last_dummy_tick = 0; GxsTokenQueue::checkRequests(); mCommentService->comment_tick(); - - return; } bool p3GxsChannels::getGroupData(const uint32_t &token, std::vector &groups) @@ -1560,6 +1559,8 @@ void p3GxsChannels::dummy_tick() } } + + cleanTimedOutSearches(); } @@ -1775,4 +1776,60 @@ bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChann return false ; } +bool p3GxsChannels::turtleSearchRequest( + const std::string& matchString, + const std::function& multiCallback, + std::time_t maxWait ) +{ + if(matchString.empty()) + { + std::cerr << __PRETTY_FUNCTION__ << " match string can't be empty!" + << std::endl; + return false; + } + TurtleRequestId sId = turtleSearchRequest(matchString); + + RS_STACK_MUTEX(mSearchCallbacksMapMutex); + mSearchCallbacksMap.emplace( + sId, + std::make_pair( + multiCallback, + std::chrono::system_clock::now() + + std::chrono::seconds(maxWait) ) ); + + return true; +} + +void p3GxsChannels::receiveDistantSearchResults( + TurtleRequestId id, const RsGxsGroupId& grpId ) +{ + std::cerr << __PRETTY_FUNCTION__ << "(" << id << ", " << grpId << ")" + << std::endl; + + RsGenExchange::receiveDistantSearchResults(id, grpId); + RsGxsGroupSummary gs; + gs.mGroupId = grpId; + netService()->retrieveDistantGroupSummary(grpId, gs); + + { + RS_STACK_MUTEX(mSearchCallbacksMapMutex); + auto cbpt = mSearchCallbacksMap.find(id); + if(cbpt != mSearchCallbacksMap.end()) + cbpt->second.first(gs); + } // end RS_STACK_MUTEX(mSearchCallbacksMapMutex); +} + +void p3GxsChannels::cleanTimedOutSearches() +{ + RS_STACK_MUTEX(mSearchCallbacksMapMutex); + auto now = std::chrono::system_clock::now(); + for( auto cbpt = mSearchCallbacksMap.begin(); + cbpt != mSearchCallbacksMap.end(); ) + if(cbpt->second.second <= now) + { + clearDistantSearchResults(cbpt->first); + cbpt = mSearchCallbacksMap.erase(cbpt); + } + else ++cbpt; +} diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index 3fb68d6dd..4f24f88b9 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -54,14 +54,14 @@ class p3GxsChannels: public RsGenExchange, public RsGxsChannels, public GxsTokenQueue, public p3Config, public RsTickEvent /* only needed for testing - remove after */ { - public: +public: + p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes, + RsGixs* gixs ); + virtual RsServiceInfo getServiceInfo(); - p3GxsChannels(RsGeneralDataService* gds, RsNetworkExchangeService* nes, RsGixs* gixs); -virtual RsServiceInfo getServiceInfo(); + virtual void service_tick(); -virtual void service_tick(); - - protected: +protected: virtual RsSerialiser* setupSerialiser(); // @see p3Config::setupSerialiser() @@ -82,7 +82,7 @@ virtual void notifyChanges(std::vector& changes); // Overloaded from RsTickEvent. virtual void handle_event(uint32_t event_type, const std::string &elabel); - public: +public: virtual bool getGroupData(const uint32_t &token, std::vector &groups); virtual bool getPostData(const uint32_t &token, std::vector &posts, std::vector &cmts); @@ -110,6 +110,18 @@ virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled); virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory); virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory); + /// @see RsGxsChannels::turtleSearchRequest + virtual bool turtleSearchRequest(const std::string& matchString, + const std::function& multiCallback, + std::time_t maxWait = 300 ); + + /** + * Receive results from turtle search @see RsGenExchange @see RsNxsObserver + * @see p3turtle::handleSearchResult + */ + void receiveDistantSearchResults( TurtleRequestId id, + const RsGxsGroupId& grpId ); + /* Comment service - Provide RsGxsCommentService - redirect to p3GxsCommentService */ virtual bool getCommentData(uint32_t token, std::vector &msgs) { return mCommentService->getGxsCommentData(token, msgs); } @@ -236,8 +248,20 @@ bool generateGroup(uint32_t &token, std::string groupName); std::vector mGenRefs; RsGxsMessageId mGenThreadId; - p3GxsCommentService *mCommentService; + p3GxsCommentService *mCommentService; std::map mKnownChannels; + + /** Store search callbacks with timeout*/ + std::map< + TurtleRequestId, + std::pair< + std::function, + std::chrono::system_clock::time_point > + > mSearchCallbacksMap; + RsMutex mSearchCallbacksMapMutex; + + /// Cleanup mSearchCallbacksMap + void cleanTimedOutSearches(); }; #endif diff --git a/libretroshare/src/util/rsjson.cc b/libretroshare/src/util/rsjson.cc new file mode 100644 index 000000000..37f6282e4 --- /dev/null +++ b/libretroshare/src/util/rsjson.cc @@ -0,0 +1,68 @@ +/******************************************************************************* + * * + * libretroshare: retroshare core library * + * * + * Copyright (C) 2018 Gioacchino Mazzurco * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "util/rsjson.h" + +#ifdef HAS_RAPIDJSON +# include +# include +# include +#else +# include +# include +# include +#endif // HAS_RAPIDJSON + + +inline int getJsonManipulatorStatePosition() +{ + static int p = std::ios_base::xalloc(); + return p; +} + +std::ostream& compactJSON(std::ostream &out) +{ + out.iword(getJsonManipulatorStatePosition()) = 1; + return out; +} + +std::ostream& prettyJSON(std::ostream &out) +{ + out.iword(getJsonManipulatorStatePosition()) = 0; + return out; +} + +std::ostream& operator<<(std::ostream &out, const RsJson &jDoc) +{ + rapidjson::StringBuffer buffer; buffer.Clear(); + if(out.iword(getJsonManipulatorStatePosition())) + { + rapidjson::Writer writer(buffer); + jDoc.Accept(writer); + } + else + { + rapidjson::PrettyWriter writer(buffer); + jDoc.Accept(writer); + } + + return out << buffer.GetString(); +} diff --git a/libretroshare/src/util/rsjson.h b/libretroshare/src/util/rsjson.h new file mode 100644 index 000000000..c52584b7e --- /dev/null +++ b/libretroshare/src/util/rsjson.h @@ -0,0 +1,56 @@ +/******************************************************************************* + * * + * libretroshare: retroshare core library * + * * + * Copyright (C) 2018 Gioacchino Mazzurco * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +#include + +#ifdef HAS_RAPIDJSON +# include +#else +# include +#endif // HAS_RAPIDJSON + +/** + * Use this type for JSON documents representations in RetroShare code + */ +typedef rapidjson::Document RsJson; + +/** + * Print out RsJson to a stream, use std::stringstream to get the string + * @param[out] out output stream + * @param[in] jDoc JSON document to print + * @return same output stream passed as out parameter + */ +std::ostream &operator<<(std::ostream &out, const RsJson &jDoc); + +/** + * Stream manipulator to print RsJson in compact format + * @param[out] out output stream + * @return same output stream passed as out parameter + */ +std::ostream& compactJSON(std::ostream &out); + +/** + * Stream manipulator to print RsJson in human readable format + * @param[out] out output stream + * @return same output stream passed as out parameter + */ +std::ostream& prettyJSON(std::ostream &out);