Implement JSON API generation for async API calls

Move JSON helpers to util/rsjson.* for better usability
Implement JSON ostream manipulator to print compact and pretty JSON
Use lambdas for API wrappers, integrate better and avoid namespace
  pollution
Removed experimental JSON API for notify client wrapper, notifications
  can be implemented automatically with moderns async API calls
Implement and automatically expose to JSON API
  RsGxsChannels::turtleSearchRequest(
        const std::string& matchString,
        const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
        std::time_t maxWait )

jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl

@@ -0,0 +1,73 @@
+/*
+ * RetroShare JSON API
+ * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+registerHandler("$%apiPath%$",
+		[$%captureVars%$](const std::shared_ptr<rb::Session> session)
+{
+	const std::multimap<std::string, std::string> headers
+	{
+		{ "Connection", "keep-alive" },
+		{ "Content-Type", "text/event-stream" }
+	};
+	session->yield(rb::OK, headers);
+
+	size_t reqSize = session->get_request()->get_header("Content-Length", 0);
+	session->fetch( reqSize, [$%captureVars%$](
+					const std::shared_ptr<rb::Session> session,
+					const rb::Bytes& body )
+	{
+		RsGenericSerializer::SerializeContext cReq(
+					nullptr, 0,
+					RsGenericSerializer::SERIALIZATION_FLAG_YIELDING );
+		RsJson& jReq(cReq.mJson);
+		jReq.Parse(reinterpret_cast<const char*>(body.data()), body.size());
+
+		RsGenericSerializer::SerializeContext cAns;
+		RsJson& jAns(cAns.mJson);
+
+		// if caller specified caller_data put it back in the answhere
+		const char kcd[] = "caller_data";
+		if(jReq.HasMember(kcd))
+			jAns.AddMember(kcd, jReq[kcd], jAns.GetAllocator());
+
+$%paramsDeclaration%$
+
+$%inputParamsDeserialization%$
+
+		$%callbackName%$ = [session]($%callbackParams%$)
+		{
+$%callbackParamsSerialization%$
+
+			std::stringstream message;
+			message << "data: " << compactJSON << ctx.mJson << "\n\n";
+			session->yield(message.str());
+			$%sessionEarlyClose%$
+		};
+
+$%functionCall%$
+
+$%outputParamsSerialization%$
+
+		// return them to the API caller
+		std::stringstream message;
+		message << "data: " << compactJSON << cAns.mJson << "\n\n";
+		session->yield(message.str());
+		$%sessionDelayedClose%$
+	} );
+});
+

jsonapi-generator/src/jsonapi-generator.cpp

@@ -25,12 +25,15 @@
 
 struct MethodParam
 {
-	MethodParam() : in(false), out(false) {}
+	MethodParam() :
+	    in(false), out(false), isMultiCallback(false), isSingleCallback(false){}
 
 	QString type;
 	QString name;
 	bool in;
 	bool out;
+	bool isMultiCallback;
+	bool isSingleCallback;
 };
 
 int main(int argc, char *argv[])
@@ -42,24 +45,18 @@ int main(int argc, char *argv[])
 	QString outputPath(argv[2]);
 	QString doxPrefix(outputPath+"/xml/");
 
-	QString wrappersDefFilePath(outputPath + "/jsonapi-wrappers.cpp");
+	QString wrappersDefFilePath(outputPath + "/jsonapi-wrappers.inl");
 	QFile wrappersDefFile(wrappersDefFilePath);
 	wrappersDefFile.remove();
 	if(!wrappersDefFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text))
 		qFatal(QString("Can't open: " + wrappersDefFilePath).toLatin1().data());
 
-	QString wrappersDeclFilePath(outputPath + "/jsonapi-wrappers.h");
+	QString cppApiIncludesFilePath(outputPath + "/jsonapi-includes.inl");
-	QFile wrappersDeclFile(wrappersDeclFilePath);
+	QFile cppApiIncludesFile(cppApiIncludesFilePath);
-	wrappersDeclFile.remove();
+	cppApiIncludesFile.remove();
-	if(!wrappersDeclFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text))
+	if(!cppApiIncludesFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text))
-		qFatal(QString("Can't open: " + wrappersDeclFilePath).toLatin1().data());
+		qFatal(QString("Can't open: " + cppApiIncludesFilePath).toLatin1().data());
-
+	QSet<QString> cppApiIncludesSet;
-	QString wrappersRegisterFilePath(outputPath + "/jsonapi-register.inl");
-	QFile wrappersRegisterFile(wrappersRegisterFilePath);
-	wrappersRegisterFile.remove();
-	if(!wrappersRegisterFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text))
-		qFatal(QString("Can't open: " + wrappersRegisterFilePath).toLatin1().data());
-
 
 	QDirIterator it(doxPrefix, QStringList() << "*8h.xml", QDir::Files);
 	while(it.hasNext())
@@ -151,21 +148,45 @@ int main(int argc, char *argv[])
 				QString retvalType = memberdef.firstChildElement("type").text();
 				QMap<QString,MethodParam> paramsMap;
 				QStringList orderedParamNames;
-				uint hasInput = false;
+				bool hasInput = false;
-				uint hasOutput = false;
+				bool hasOutput = false;
+				bool hasSingleCallback = false;
+				bool hasMultiCallback = false;
+				QString callbackName;
+				QString callbackParams;
 
 				QDomNodeList params = memberdef.elementsByTagName("param");
 				for (int k = 0; k < params.size(); ++k)
 				{
 					QDomElement tmpPE = params.item(k).toElement();
 					MethodParam tmpParam;
-					tmpParam.name = tmpPE.firstChildElement("declname").text();
+					QString& pName(tmpParam.name);
-					QDomElement tmpType = tmpPE.firstChildElement("type");
 					QString& pType(tmpParam.type);
+					pName = tmpPE.firstChildElement("declname").text();
+					QDomElement tmpType = tmpPE.firstChildElement("type");
 					pType = tmpType.text();
 					if(pType.startsWith("const ")) pType.remove(0,6);
-					pType.replace(QString("&"), QString());
+					if(pType.startsWith("std::function"))
-					pType.replace(QString(" "), QString());
+					{
+						if(pType.endsWith('&')) pType.chop(1);
+						if(pName.startsWith("multiCallback"))
+						{
+							tmpParam.isMultiCallback = true;
+							hasMultiCallback = true;
+						}
+						else if(pName.startsWith("callback"))
+						{
+							tmpParam.isSingleCallback = true;
+							hasSingleCallback = true;
+						}
+						callbackName = pName;
+						callbackParams = pType;
+					}
+					else
+					{
+						pType.replace(QString("&"), QString());
+						pType.replace(QString(" "), QString());
+					}
 					paramsMap.insert(tmpParam.name, tmpParam);
 					orderedParamNames.push_back(tmpParam.name);
 				}
@@ -242,6 +263,58 @@ int main(int argc, char *argv[])
 					        "\t\t\tRS_SERIAL_PROCESS(retval);\n";
 				if(hasOutput) outputParamsSerialization += "\t\t}\n";
 
+				QString captureVars;
+
+				QString sessionEarlyClose;
+				if(hasSingleCallback)
+					sessionEarlyClose = "session->close();";
+
+				QString sessionDelayedClose;
+				if(hasMultiCallback)
+				{
+					sessionDelayedClose = "mService.schedule( [session](){session->close();}, std::chrono::seconds(maxWait+120) );";
+					captureVars = "this";
+				}
+
+				QString callbackParamsSerialization;
+
+				if(hasSingleCallback || hasMultiCallback ||
+				        ((callbackParams.indexOf('(')+2) < callbackParams.indexOf(')')))
+				{
+					QString& cbs(callbackParamsSerialization);
+
+					callbackParams = callbackParams.split('(')[1];
+					callbackParams = callbackParams.split(')')[0];
+
+					cbs += "RsGenericSerializer::SerializeContext ctx;\n";
+
+					for (QString cbPar : callbackParams.split(','))
+					{
+						bool isConst(cbPar.startsWith("const "));
+						QChar pSep(' ');
+						bool isRef(cbPar.contains('&'));
+						if(isRef) pSep = '&';
+						int sepIndex = cbPar.lastIndexOf(pSep)+1;
+						QString cpt(cbPar.mid(0, sepIndex));
+						cpt.remove(0,6);
+						QString cpn(cbPar.mid(sepIndex));
+
+						cbs += "\t\t\tRsTypeSerializer::serial_process(";
+						cbs += "RsGenericSerializer::TO_JSON, ctx, ";
+						if(isConst)
+						{
+							cbs += "const_cast<";
+							cbs += cpt;
+							cbs += ">(";
+						}
+						cbs += cpn;
+						if(isConst) cbs += ")";
+						cbs += ", \"";
+						cbs += cpn;
+						cbs += "\" );\n";
+					}
+				}
+
 				QMap<QString,QString> substitutionsMap;
 				substitutionsMap.insert("paramsDeclaration", paramsDeclaration);
 				substitutionsMap.insert("inputParamsDeserialization", inputParamsDeserialization);
@@ -249,8 +322,20 @@ int main(int argc, char *argv[])
 				substitutionsMap.insert("wrapperName", wrapperName);
 				substitutionsMap.insert("headerFileName", headerFileName);
 				substitutionsMap.insert("functionCall", functionCall);
+				substitutionsMap.insert("apiPath", apiPath);
+				substitutionsMap.insert("sessionEarlyClose", sessionEarlyClose);
+				substitutionsMap.insert("sessionDelayedClose", sessionDelayedClose);
+				substitutionsMap.insert("captureVars", captureVars);
+				substitutionsMap.insert("callbackName", callbackName);
+				substitutionsMap.insert("callbackParams", callbackParams);
+				substitutionsMap.insert("callbackParamsSerialization", callbackParamsSerialization);
 
-				QFile templFile(sourcePath + "/method-wrapper-template.cpp.tmpl");
+				QString templFilePath(sourcePath);
+				if(hasMultiCallback || hasSingleCallback)
+					templFilePath.append("/async-method-wrapper-template.cpp.tmpl");
+				else templFilePath.append("/method-wrapper-template.cpp.tmpl");
+
+				QFile templFile(templFilePath);
 				templFile.open(QIODevice::ReadOnly);
 				QString wrapperDef(templFile.readAll());
 
@@ -261,15 +346,13 @@ int main(int argc, char *argv[])
 
 				wrappersDefFile.write(wrapperDef.toLocal8Bit());
 
-				QString wrapperDecl("void " + instanceName + methodName + "Wrapper(const std::shared_ptr<rb::Session> session);\n");
+				cppApiIncludesSet.insert("#include \"retroshare/" + headerFileName + "\"\n");
-				wrappersDeclFile.write(wrapperDecl.toLocal8Bit());
-
-
-				QString wrapperReg("registerHandler(\""+apiPath+"\", "+wrapperName+");\n");
-				wrappersRegisterFile.write(wrapperReg.toLocal8Bit());
 			}
 		}
 	}
 
+	for(const QString& incl : cppApiIncludesSet)
+		cppApiIncludesFile.write(incl.toLocal8Bit());
+
 	return 0;
 }

jsonapi-generator/src/method-wrapper-template.cpp.tmpl

@@ -16,14 +16,8 @@
  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <sstream>
+registerHandler("$%apiPath%$",
-#include <memory>
+				[$%captureVars%$](const std::shared_ptr<rb::Session> session)
-#include <restbed>
-#include "retroshare/$%headerFileName%$"
-
-namespace rb = restbed;
-
-void $%wrapperName%$(const std::shared_ptr<rb::Session> session)
 {
 	size_t reqSize = session->get_request()->get_header("Content-Length", 0);
 	session->fetch( reqSize, [](
@@ -66,5 +60,5 @@ $%outputParamsSerialization%$
 		};
 		session->close(rb::OK, ans, headers);
 	} );
-}
+});
 

libretroshare/src/jsonapi/jsonapi.cpp

@@ -18,16 +18,19 @@
 
 #include "jsonapi.h"
 
-#include <rapid_json/document.h>
+#include <sstream>
-#include <rapid_json/writer.h>
+#include <memory>
-#include <rapid_json/stringbuffer.h>
+#include <restbed>
+
+#include "util/rsjson.h"
+#include "retroshare/rsgxschannels.h"
 
 // Generated at compile time
-#include "jsonapi-wrappers.h"
+#include "jsonapi-includes.inl"
 
 JsonApiServer::JsonApiServer(
         uint16_t port, const std::function<void(int)> shutdownCallback) :
-    mPort(port), mShutdownCallback(shutdownCallback), notifyClientWrapper(*this)
+    mPort(port), mShutdownCallback(shutdownCallback)
 {
 	registerHandler("/jsonApiServer/shutdown",
 	                [this](const std::shared_ptr<rb::Session>)
@@ -35,32 +38,16 @@ JsonApiServer::JsonApiServer(
 		shutdown();
 	});
 
-	registerHandler("/jsonApiServer/notifications",
-	        [this](const std::shared_ptr<rb::Session> session)
-	{
-		const auto headers = std::multimap<std::string, std::string>
-		{
-			{ "Connection", "keep-alive" },
-			{ "Cache-Control", "no-cache" },
-			{ "Content-Type", "text/event-stream" },
-		};
-
-		session->yield(rb::OK, headers,
-		               [this](const std::shared_ptr<rb::Session> session)
-		{
-			notifySessions.push_back(session);
-		} );
-	} );
-
 // Generated at compile time
-#include "jsonapi-register.inl"
+#include "jsonapi-wrappers.inl"
 }
 
 void JsonApiServer::run()
 {
 	std::shared_ptr<rb::Settings> settings(new rb::Settings);
 	settings->set_port(mPort);
-	settings->set_default_header("Connection", "close");
+//	settings->set_default_header("Connection", "close");
+	settings->set_default_header("Cache-Control", "no-cache");
 	mService.start(settings);
 }
 
@@ -80,47 +67,3 @@ void JsonApiServer::shutdown(int exitCode)
 	mService.stop();
 	mShutdownCallback(exitCode);
 }
-
-void JsonApiServer::cleanClosedNotifySessions()
-{
-	notifySessions.erase(
-	            std::remove_if(
-	                notifySessions.begin(), notifySessions.end(),
-	                [](const std::shared_ptr<rb::Session> &s)
-	{ return s->is_closed(); } ), notifySessions.end());
-}
-
-JsonApiServer::NotifyClientWrapper::NotifyClientWrapper(JsonApiServer& parent) :
-    NotifyClient(), mJsonApiServer(parent)
-{
-	rsNotify->registerNotifyClient(static_cast<NotifyClient*>(this));
-}
-
-void JsonApiServer::NotifyClientWrapper::notifyTurtleSearchResult(
-        uint32_t searchId, const std::list<TurtleFileInfo>& files )
-{
-	mJsonApiServer.cleanClosedNotifySessions();
-
-	RsGenericSerializer::SerializeContext cAns;
-	RsJson& jAns(cAns.mJson);
-
-	// serialize parameters and method name to JSON
-	{
-		std::string methodName("NotifyClient/notifyTurtleSearchResult");
-		std::list<TurtleFileInfo> filesCopy(files);
-		RsGenericSerializer::SerializeContext& ctx(cAns);
-		RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON);
-		RS_SERIAL_PROCESS(methodName);
-		RS_SERIAL_PROCESS(searchId);
-//		RS_SERIAL_PROCESS(filesCopy);
-	}
-
-	rapidjson::StringBuffer buffer;
-	rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
-	jAns.Accept(writer);
-	std::string message(buffer.GetString(), buffer.GetSize());
-	message.insert(0, "data: "); message.append("\n\n");
-
-	for(auto session : mJsonApiServer.notifySessions)
-		session->yield(message);
-}

libretroshare/src/jsonapi/jsonapi.h

@@ -21,7 +21,6 @@
 #include <restbed>
 
 #include "util/rsthreads.h"
-#include "retroshare/rsnotify.h"
 
 namespace rb = restbed;
 
@@ -70,20 +69,5 @@ private:
 	uint16_t mPort;
 	rb::Service mService;
 	const std::function<void(int)> mShutdownCallback;
-
-	std::list<std::shared_ptr<rb::Session> > notifySessions;
-	void cleanClosedNotifySessions();
-
-	struct NotifyClientWrapper : NotifyClient
-	{
-		NotifyClientWrapper(JsonApiServer& parent);
-
-		void notifyTurtleSearchResult(
-		        uint32_t searchId, const std::list<TurtleFileInfo>& files);
-
-	private:
-		JsonApiServer& mJsonApiServer;
-	};
-	NotifyClientWrapper notifyClientWrapper;
 };
 

libretroshare/src/libretroshare.pro

@@ -776,10 +776,12 @@ SOURCES += gxstunnel/p3gxstunnel.cc \
 # new serialization code
 HEADERS += serialiser/rsserializable.h \
            serialiser/rsserializer.h \
-           serialiser/rstypeserializer.h
+           serialiser/rstypeserializer.h \
+           util/rsjson.h
 
 SOURCES += serialiser/rsserializer.cc \
-           serialiser/rstypeserializer.cc 
+           serialiser/rstypeserializer.cc \
+           util/rsjson.cc
 
 # Identity Service
 HEADERS += retroshare/rsidentity.h \
@@ -874,9 +876,8 @@ rs_jsonapi {
     DOXIGEN_INPUT_DIRECTORY=$$system_path($$clean_path($${PWD}))
     DOXIGEN_CONFIG_SRC=$$system_path($$clean_path($${RS_SRC_PATH}/jsonapi-generator/src/jsonapi-generator-doxygen.conf))
     DOXIGEN_CONFIG_OUT=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-generator-doxygen.conf))
-    WRAPPERS_DEF_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-wrappers.cpp))
+    WRAPPERS_INCL_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-includes.inl))
-    WRAPPERS_DECL_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-wrappers.h))
+    WRAPPERS_REG_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-wrappers.inl))
-    WRAPPERS_REG_FILE=$$system_path($$clean_path($${JSONAPI_GENERATOR_OUT}/jsonapi-register.inl))
 
     restbed.target = $$system_path($$clean_path($${RESTBED_BUILD_PATH}/library/librestbed.a))
     restbed.commands = \
@@ -890,30 +891,22 @@ rs_jsonapi {
 
     PRE_TARGETDEPS *= $${JSONAPI_GENERATOR_EXE}
     INCLUDEPATH *= $${JSONAPI_GENERATOR_OUT}
-    GENERATED_HEADERS += $${WRAPPERS_DECL_FILE} $${WRAPPERS_REG_FILE}
+    GENERATED_HEADERS += $${WRAPPERS_INCL_FILE}
-    GENERATED_SOURCES += $${WRAPPERS_DEF_FILE}
 
-    jsonwrappersdecl.target = $${WRAPPERS_DECL_FILE}
+    jsonwrappersincl.target = $${WRAPPERS_INCL_FILE}
-    jsonwrappersdecl.commands = \
+    jsonwrappersincl.commands = \
         cp $${DOXIGEN_CONFIG_SRC} $${DOXIGEN_CONFIG_OUT}; \
         echo OUTPUT_DIRECTORY=$${JSONAPI_GENERATOR_OUT} >> $${DOXIGEN_CONFIG_OUT};\
         echo INPUT=$${DOXIGEN_INPUT_DIRECTORY} >> $${DOXIGEN_CONFIG_OUT}; \
         doxygen $${DOXIGEN_CONFIG_OUT}; \
         $${JSONAPI_GENERATOR_EXE} $${JSONAPI_GENERATOR_SRC} $${JSONAPI_GENERATOR_OUT};
-    QMAKE_EXTRA_TARGETS += jsonwrappersdecl
+    QMAKE_EXTRA_TARGETS += jsonwrappersincl
-    libretroshare.depends += jsonwrappersdecl
+    libretroshare.depends += jsonwrappersincl
-    PRE_TARGETDEPS *= $${WRAPPERS_DECL_FILE}
+    PRE_TARGETDEPS *= $${WRAPPERS_INCL_FILE}
-
-    jsonwrappersdef.target = $${WRAPPERS_DEF_FILE}
-    jsonwrappersdef.commands = touch $${WRAPPERS_DEF_FILE}
-    jsonwrappersdef.depends = jsonwrappersdecl
-    QMAKE_EXTRA_TARGETS += jsonwrappersdef
-    libretroshare.depends += jsonwrappersdef
-    PRE_TARGETDEPS *= $${WRAPPERS_DEF_FILE}
 
     jsonwrappersreg.target = $${WRAPPERS_REG_FILE}
     jsonwrappersreg.commands = touch $${WRAPPERS_REG_FILE}
-    jsonwrappersreg.depends = jsonwrappersdef
+    jsonwrappersreg.depends = jsonwrappersincl
     QMAKE_EXTRA_TARGETS += jsonwrappersreg
     libretroshare.depends += jsonwrappersreg
     PRE_TARGETDEPS *= $${WRAPPERS_REG_FILE}

libretroshare/src/retroshare/rsgxschannels.h

@@ -25,6 +25,7 @@
 #include <inttypes.h>
 #include <string>
 #include <list>
+#include <functional>
 
 #include "retroshare/rstokenservice.h"
 #include "retroshare/rsgxsifacehelper.h"
@@ -266,6 +267,20 @@ public:
 	 */
 	virtual bool ExtraFileRemove(const RsFileHash& hash) = 0;
 
+	/**
+	 * @brief Request remote channels search
+	 * @jsonapi{development}
+	 * @param[in] matchString string to look for in the search
+	 * @param multiCallback function that will be called each time a search
+	 * result is received
+	 * @param[in] maxWait maximum wait time in seconds for search results
+	 * @return false on error, true otherwise
+	 */
+	virtual bool turtleSearchRequest(
+	        const std::string& matchString,
+	        const std::function<void (const RsGxsGroupSummary& result)>& multiCallback,
+	        std::time_t maxWait = 300 ) = 0;
+
 	//////////////////////////////////////////////////////////////////////////////
     ///                     Distant synchronisation methods                    ///
     //////////////////////////////////////////////////////////////////////////////

libretroshare/src/retroshare/rsgxsiface.h

@@ -32,6 +32,7 @@
 #include "gxs/rsgxsdata.h"
 #include "retroshare/rsgxsifacetypes.h"
 #include "util/rsdeprecate.h"
+#include "serialiser/rsserializable.h"
 
 /*!
  * This structure is used to transport group summary information when a GXS

libretroshare/src/serialiser/rsserializer.h

@@ -151,15 +151,11 @@
 #include <string.h>
 #include <iostream>
 #include <string>
-#ifdef HAS_RAPIDJSON
-#include <rapidjson/document.h>
-#else
-#include <rapid_json/document.h>
-#endif // HAS_RAPIDJSON
 
 #include "retroshare/rsflags.h"
 #include "serialiser/rsserial.h"
 #include "util/rsdeprecate.h"
+#include "util/rsjson.h"
 
 struct RsItem;
 
@@ -198,8 +194,6 @@ class RsRawSerialiser: public RsSerialType
 		virtual	RsItem *    deserialise(void *data, uint32_t *size);
 };
 
-typedef rapidjson::Document RsJson;
-
 /// Top class for all services and config serializers.
 struct RsGenericSerializer : RsSerialType
 {

libretroshare/src/serialiser/rstypeserializer.cc

@@ -660,16 +660,3 @@ bool RsTypeSerializer::from_JSON( const std::string& /*memberName*/,
                                   RsTypeSerializer::TlvMemBlock_proxy&,
                                   RsJson& /*jDoc*/)
 { return true; }
-
-
-//============================================================================//
-//                      RsJson std:ostream support                            //
-//============================================================================//
-
-std::ostream &operator<<(std::ostream &out, const RsJson &jDoc)
-{
-	rapidjson::StringBuffer buffer; buffer.Clear();
-	rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
-	jDoc.Accept(writer);
-	return out << buffer.GetString();
-}

libretroshare/src/serialiser/rstypeserializer.h

@@ -31,12 +31,8 @@
 
 #include "serialiser/rsserializer.h"
 #include "serialiser/rsserializable.h"
+#include "util/rsjson.h"
 
-#ifdef HAS_RAPIDJSON
-#include <rapidjson/document.h>
-#else
-#include <rapid_json/document.h>
-#endif // HAS_RAPIDJSON
 #include <typeinfo> // for typeid
 #include <type_traits>
 #include <errno.h>
@@ -116,7 +112,6 @@
 \
 } while(false)
 
-std::ostream &operator<<(std::ostream &out, const RsJson &jDoc);
 
 struct RsTypeSerializer
 {

libretroshare/src/services/p3gxschannels.cc

@@ -69,7 +69,8 @@ p3GxsChannels::p3GxsChannels(
         RsGixs* gixs ) :
     RsGenExchange( gds, nes, new RsGxsChannelSerialiser(),
                    RS_SERVICE_GXS_TYPE_CHANNELS, gixs, channelsAuthenPolicy() ),
-    RsGxsChannels(static_cast<RsGxsIface&>(*this)), GxsTokenQueue(this)
+    RsGxsChannels(static_cast<RsGxsIface&>(*this)), GxsTokenQueue(this),
+    mSearchCallbacksMapMutex("GXS channels search")
 {
 	// For Dummy Msgs.
 	mGenActive = false;
@@ -351,8 +352,6 @@ static  time_t last_dummy_tick = 0;
 	GxsTokenQueue::checkRequests();
 
 	mCommentService->comment_tick();
-
-	return;
 }
 
 bool p3GxsChannels::getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups)
@@ -1560,6 +1559,8 @@ void p3GxsChannels::dummy_tick()
 		}
 
 	}
+
+	cleanTimedOutSearches();
 }
 
 
@@ -1775,4 +1776,60 @@ bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChann
         return false ;
 }
 
+bool p3GxsChannels::turtleSearchRequest(
+        const std::string& matchString,
+        const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
+        std::time_t maxWait )
+{
+	if(matchString.empty())
+	{
+		std::cerr << __PRETTY_FUNCTION__ << " match string can't be empty!"
+		          << std::endl;
+		return false;
+	}
 
+	TurtleRequestId sId = turtleSearchRequest(matchString);
+
+	RS_STACK_MUTEX(mSearchCallbacksMapMutex);
+	mSearchCallbacksMap.emplace(
+	            sId,
+	            std::make_pair(
+	                multiCallback,
+	                std::chrono::system_clock::now() +
+	                    std::chrono::seconds(maxWait) ) );
+
+	return true;
+}
+
+void p3GxsChannels::receiveDistantSearchResults(
+        TurtleRequestId id, const RsGxsGroupId& grpId )
+{
+	std::cerr << __PRETTY_FUNCTION__ << "(" << id << ", " << grpId << ")"
+	          << std::endl;
+
+	RsGenExchange::receiveDistantSearchResults(id, grpId);
+	RsGxsGroupSummary gs;
+	gs.mGroupId = grpId;
+	netService()->retrieveDistantGroupSummary(grpId, gs);
+
+	{
+		RS_STACK_MUTEX(mSearchCallbacksMapMutex);
+		auto cbpt = mSearchCallbacksMap.find(id);
+		if(cbpt != mSearchCallbacksMap.end())
+			cbpt->second.first(gs);
+	} // end RS_STACK_MUTEX(mSearchCallbacksMapMutex);
+}
+
+void p3GxsChannels::cleanTimedOutSearches()
+{
+	RS_STACK_MUTEX(mSearchCallbacksMapMutex);
+	auto now = std::chrono::system_clock::now();
+	for( auto cbpt = mSearchCallbacksMap.begin();
+	     cbpt != mSearchCallbacksMap.end(); )
+		if(cbpt->second.second <= now)
+		{
+			clearDistantSearchResults(cbpt->first);
+			cbpt = mSearchCallbacksMap.erase(cbpt);
+		}
+		else ++cbpt;
+}

libretroshare/src/services/p3gxschannels.h

@@ -54,14 +54,14 @@ class p3GxsChannels: public RsGenExchange, public RsGxsChannels,
 	public GxsTokenQueue, public p3Config,
 	public RsTickEvent	/* only needed for testing - remove after */
 {
-	public:
+public:
+	p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes,
+	               RsGixs* gixs );
+	virtual RsServiceInfo getServiceInfo();
 
-	p3GxsChannels(RsGeneralDataService* gds, RsNetworkExchangeService* nes, RsGixs* gixs);
+	virtual void service_tick();
-virtual RsServiceInfo getServiceInfo();
 
-virtual void service_tick();
+protected:
-
-	protected:
 
 
 	virtual RsSerialiser* setupSerialiser();                            // @see p3Config::setupSerialiser()
@@ -82,7 +82,7 @@ virtual void notifyChanges(std::vector<RsGxsNotify*>& changes);
         // Overloaded from RsTickEvent.
 virtual void handle_event(uint32_t event_type, const std::string &elabel);
 
-	public:
+public:
 
 virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups);
 virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts);
@@ -110,6 +110,18 @@ virtual	bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled);
 virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory);
 virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory);
 
+	/// @see RsGxsChannels::turtleSearchRequest
+	virtual bool turtleSearchRequest(const std::string& matchString,
+	        const std::function<void (const RsGxsGroupSummary&)>& multiCallback,
+	        std::time_t maxWait = 300 );
+
+	/**
+	 * Receive results from turtle search @see RsGenExchange @see RsNxsObserver
+	 * @see p3turtle::handleSearchResult
+	 */
+	void receiveDistantSearchResults( TurtleRequestId id,
+	                                  const RsGxsGroupId& grpId );
+
 	/* Comment service - Provide RsGxsCommentService - redirect to p3GxsCommentService */
 	virtual bool getCommentData(uint32_t token, std::vector<RsGxsComment> &msgs)
 	{ return mCommentService->getGxsCommentData(token, msgs); }
@@ -236,8 +248,20 @@ bool generateGroup(uint32_t &token, std::string groupName);
 	std::vector<ChannelDummyRef> mGenRefs;
 	RsGxsMessageId mGenThreadId;
 
-	p3GxsCommentService *mCommentService;	
+	p3GxsCommentService *mCommentService;
     std::map<RsGxsGroupId,time_t> mKnownChannels;
+
+	/** Store search callbacks with timeout*/
+	std::map<
+	    TurtleRequestId,
+	    std::pair<
+	        std::function<void (const RsGxsGroupSummary&)>,
+	        std::chrono::system_clock::time_point >
+	 > mSearchCallbacksMap;
+	RsMutex mSearchCallbacksMapMutex;
+
+	/// Cleanup mSearchCallbacksMap
+	void cleanTimedOutSearches();
 };
 
 #endif 

libretroshare/src/util/rsjson.cc

@@ -0,0 +1,68 @@
+/*******************************************************************************
+ *                                                                             *
+ * libretroshare: retroshare core library                                      *
+ *                                                                             *
+ * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Lesser General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Lesser General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Lesser General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+
+#include "util/rsjson.h"
+
+#ifdef HAS_RAPIDJSON
+#	include <rapidjson/writer.h>
+#	include <rapidjson/stringbuffer.h>
+#	include <rapidjson/prettywriter.h>
+#else
+#	include <rapid_json/writer.h>
+#	include <rapid_json/stringbuffer.h>
+#	include <rapid_json/prettywriter.h>
+#endif // HAS_RAPIDJSON
+
+
+inline int getJsonManipulatorStatePosition()
+{
+	static int p = std::ios_base::xalloc();
+	return p;
+}
+
+std::ostream& compactJSON(std::ostream &out)
+{
+	out.iword(getJsonManipulatorStatePosition()) = 1;
+	return out;
+}
+
+std::ostream& prettyJSON(std::ostream &out)
+{
+	out.iword(getJsonManipulatorStatePosition()) = 0;
+	return out;
+}
+
+std::ostream& operator<<(std::ostream &out, const RsJson &jDoc)
+{
+	rapidjson::StringBuffer buffer; buffer.Clear();
+	if(out.iword(getJsonManipulatorStatePosition()))
+	{
+		rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+		jDoc.Accept(writer);
+	}
+	else
+	{
+		rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(buffer);
+		jDoc.Accept(writer);
+	}
+
+	return out << buffer.GetString();
+}

libretroshare/src/util/rsjson.h

@@ -0,0 +1,56 @@
+/*******************************************************************************
+ *                                                                             *
+ * libretroshare: retroshare core library                                      *
+ *                                                                             *
+ * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>                  *
+ *                                                                             *
+ * This program is free software: you can redistribute it and/or modify        *
+ * it under the terms of the GNU Lesser General Public License as              *
+ * published by the Free Software Foundation, either version 3 of the          *
+ * License, or (at your option) any later version.                             *
+ *                                                                             *
+ * This program is distributed in the hope that it will be useful,             *
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of              *
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the                *
+ * GNU Lesser General Public License for more details.                         *
+ *                                                                             *
+ * You should have received a copy of the GNU Lesser General Public License    *
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
+ *                                                                             *
+ *******************************************************************************/
+#pragma once
+
+#include <iostream>
+
+#ifdef HAS_RAPIDJSON
+#	include <rapidjson/document.h>
+#else
+#	include <rapid_json/document.h>
+#endif // HAS_RAPIDJSON
+
+/**
+ * Use this type for JSON documents representations in RetroShare code
+ */
+typedef rapidjson::Document RsJson;
+
+/**
+ * Print out RsJson to a stream, use std::stringstream to get the string
+ * @param[out] out output stream
+ * @param[in] jDoc JSON document to print
+ * @return same output stream passed as out parameter
+ */
+std::ostream &operator<<(std::ostream &out, const RsJson &jDoc);
+
+/**
+ * Stream manipulator to print RsJson in compact format
+ * @param[out] out output stream
+ * @return same output stream passed as out parameter
+ */
+std::ostream& compactJSON(std::ostream &out);
+
+/**
+ * Stream manipulator to print RsJson in human readable format
+ * @param[out] out output stream
+ * @return same output stream passed as out parameter
+ */
+std::ostream& prettyJSON(std::ostream &out);