Implement automatic JSON API generation

qmake file add jsonapi-generator target to compile JSON API generator qmake files add rs_jsonapi CONFIG option to enable/disable JSON API at compile time RsTypeSerializer pass down same serialization flags when creating new context for nested objects serial job RsGxsChannels expose a few methods through JSON API as example Derive a few GXS types (RsGxsChannelGroup, RsGxsChannelPost, RsGxsFile, RsMsgMetaData) from RsSerializables so they can be used for the JSON API Create RsGenericSerializer::SERIALIZATION_FLAG_YIELDING so JSON objects that miss some fields can be still deserialized, this improve API usability SerializeContext offer friendly constructor with default paramethers Add restbed 4.6 library as git submodule as most systems doesn't have it yet Add a bit of documentation about JSON API into jsonapi-generator/README.adoc Add JsonApiServer class to expose the JSON API via HTTP protocol
2025-05-02 06:06:10 -04:00 · 2018-06-23 17:13:38 +02:00 · 2018-06-23 17:13:38 +02:00 · 7ad337c8d2
commit 7ad337c8d2
parent 2f159efb10
22 changed files with 1205 additions and 83 deletions
--- a/jsonapi-generator/src/jsonapi-generator-doxygen.conf
+++ b/jsonapi-generator/src/jsonapi-generator-doxygen.conf
@ -0,0 +1,230 @@
+DOXYFILE_ENCODING      = UTF-8
+PROJECT_NAME           = "libretroshare"
+#OUTPUT_DIRECTORY       =
+
+ALIASES += jsonapi{1}="\xmlonly<jsonapi minversion=\"\1\"/>\endxmlonly"
+
+# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the
+# documentation from any documented member that it re-implements.
+# The default value is: YES.
+
+INHERIT_DOCS           = YES
+
+# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments
+# according to the Markdown format, which allows for more readable
+# documentation. See http://daringfireball.net/projects/markdown/ for details.
+# The output of markdown processing is further processed by doxygen, so you can
+# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in
+# case of backward compatibilities issues.
+# The default value is: YES.
+
+MARKDOWN_SUPPORT       = YES
+
+# When enabled doxygen tries to link words that correspond to documented
+# classes, or namespaces to their corresponding documentation. Such a link can
+# be prevented in individual cases by putting a % sign in front of the word or
+# globally by setting AUTOLINK_SUPPORT to NO.
+# The default value is: YES.
+
+AUTOLINK_SUPPORT       = YES
+
+
+# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in
+# documentation are documented, even if no documentation was available. Private
+# class members and static file members will be hidden unless the
+# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES.
+# Note: This will also disable the warnings about undocumented members that are
+# normally produced when WARNINGS is set to YES.
+# The default value is: NO.
+
+EXTRACT_ALL            = YES
+
+
+# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all
+# undocumented members inside documented classes or files. If set to NO these
+# members will be included in the various overviews, but no documentation
+# section is generated. This option has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_MEMBERS     = NO
+
+# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all
+# undocumented classes that are normally visible in the class hierarchy. If set
+# to NO, these classes will be included in the various overviews. This option
+# has no effect if EXTRACT_ALL is enabled.
+# The default value is: NO.
+
+HIDE_UNDOC_CLASSES     = NO
+
+
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+
+# The INPUT tag is used to specify the files and/or directories that contain
+# documented source files. You may enter file names like myfile.cpp or
+# directories like /usr/src/myproject. Separate the files or directories with
+# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING
+# Note: If this tag is empty the current directory is searched.
+
+#INPUT                  =
+
+# This tag can be used to specify the character encoding of the source files
+# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses
+# libiconv (or the iconv built into libc) for the transcoding. See the libiconv
+# documentation (see: https://www.gnu.org/software/libiconv/) for the list of
+# possible encodings.
+# The default value is: UTF-8.
+
+INPUT_ENCODING         = UTF-8
+
+# If the value of the INPUT tag contains directories, you can use the
+# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and
+# *.h) to filter out the source-files in the directories.
+#
+# Note that for custom extensions or not directly supported extensions you also
+# need to set EXTENSION_MAPPING for the extension otherwise the files are not
+# read by doxygen.
+#
+# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp,
+# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h,
+# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc,
+# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08,
+# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf.
+
+FILE_PATTERNS          = *.c \
+                         *.cc \
+                         *.cxx \
+                         *.cpp \
+                         *.c++ \
+                         *.java \
+                         *.ii \
+                         *.ixx \
+                         *.ipp \
+                         *.i++ \
+                         *.inl \
+                         *.idl \
+                         *.ddl \
+                         *.odl \
+                         *.h \
+                         *.hh \
+                         *.hxx \
+                         *.hpp \
+                         *.h++ \
+                         *.cs \
+                         *.d \
+                         *.php \
+                         *.php4 \
+                         *.php5 \
+                         *.phtml \
+                         *.inc \
+                         *.m \
+                         *.markdown \
+                         *.md \
+                         *.mm \
+                         *.dox \
+                         *.py \
+                         *.pyw \
+                         *.f90 \
+                         *.f95 \
+                         *.f03 \
+                         *.f08 \
+                         *.f \
+                         *.for \
+                         *.tcl \
+                         *.vhd \
+                         *.vhdl \
+                         *.ucf \
+                         *.qsf
+
+# The RECURSIVE tag can be used to specify whether or not subdirectories should
+# be searched for input files as well.
+# The default value is: NO.
+
+RECURSIVE              = YES
+
+# The EXCLUDE tag can be used to specify files and/or directories that should be
+# excluded from the INPUT source files. This way you can easily exclude a
+# subdirectory from a directory tree whose root is specified with the INPUT tag.
+#
+# Note that relative paths are relative to the directory from which doxygen is
+# run.
+
+EXCLUDE                =
+
+# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or
+# directories that are symbolic links (a Unix file system feature) are excluded
+# from the input.
+# The default value is: NO.
+
+EXCLUDE_SYMLINKS       = NO
+
+# If the value of the INPUT tag contains directories, you can use the
+# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude
+# certain files from those directories.
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories for example use the pattern */test/*
+
+EXCLUDE_PATTERNS       =
+
+# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names
+# (namespaces, classes, functions, etc.) that should be excluded from the
+# output. The symbol name can be a fully qualified name, a word, or if the
+# wildcard * is used, a substring. Examples: ANamespace, AClass,
+# AClass::ANamespace, ANamespace::*Test
+#
+# Note that the wildcards are matched against the file with absolute path, so to
+# exclude all test directories use the pattern */test/*
+
+EXCLUDE_SYMBOLS        =
+
+
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+
+# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output
+# The default value is: YES.
+
+GENERATE_HTML          = NO
+
+# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output.
+# The default value is: YES.
+
+GENERATE_LATEX         = NO
+
+# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that
+# captures the structure of the code including all documentation.
+# The default value is: NO.
+
+GENERATE_XML           = YES
+
+
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+
+# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all
+# C-preprocessor directives found in the sources and include files.
+# The default value is: YES.
+
+ENABLE_PREPROCESSING   = YES
+
+# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names
+# in the source code. If set to NO, only conditional compilation will be
+# performed. Macro expansion can be done in a controlled way by setting
+# EXPAND_ONLY_PREDEF to YES.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+MACRO_EXPANSION        = NO
+
+# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then
+# the macro expansion is limited to the macros specified with the PREDEFINED and
+# EXPAND_AS_DEFINED tags.
+# The default value is: NO.
+# This tag requires that the tag ENABLE_PREPROCESSING is set to YES.
+
+EXPAND_ONLY_PREDEF     = NO
+
--- a/jsonapi-generator/src/jsonapi-generator.cpp
+++ b/jsonapi-generator/src/jsonapi-generator.cpp
@ -0,0 +1,239 @@
+/*
+ * RetroShare JSON API
+ * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <QCoreApplication>
+#include <QDebug>
+#include <QtXml>
+#include <QDirIterator>
+#include <QFileInfo>
+#include <iterator>
+
+struct MethodParam
+{
+	MethodParam() : in(false), out(false) {}
+
+	QString type;
+	QString name;
+	bool in;
+	bool out;
+};
+
+int main(int argc, char *argv[])
+{
+	if(argc != 3)
+		qFatal("Usage: jsonapi-generator SOURCE_PATH OUTPUT_PATH");
+
+	QString sourcePath(argv[1]);
+	QString outputPath(argv[2]);
+	QString doxPrefix(outputPath+"/xml/");
+
+	QString wrappersDefFilePath(outputPath + "/jsonapi-wrappers.cpp");
+	QFile wrappersDefFile(wrappersDefFilePath);
+	wrappersDefFile.remove();
+	if(!wrappersDefFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text))
+		qFatal(QString("Can't open: " + wrappersDefFilePath).toLatin1().data());
+
+	QString wrappersDeclFilePath(outputPath + "/jsonapi-wrappers.h");
+	QFile wrappersDeclFile(wrappersDeclFilePath);
+	wrappersDeclFile.remove();
+	if(!wrappersDeclFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text))
+		qFatal(QString("Can't open: " + wrappersDeclFilePath).toLatin1().data());
+
+	QString wrappersRegisterFilePath(outputPath + "/jsonapi-register.inl");
+	QFile wrappersRegisterFile(wrappersRegisterFilePath);
+	wrappersRegisterFile.remove();
+	if(!wrappersRegisterFile.open(QIODevice::WriteOnly|QIODevice::Append|QIODevice::Text))
+		qFatal(QString("Can't open: " + wrappersRegisterFilePath).toLatin1().data());
+
+
+	QDirIterator it(doxPrefix, QStringList() << "*8h.xml", QDir::Files);
+	while(it.hasNext())
+	{
+		QDomDocument hDoc;
+		QString hFilePath(it.next());
+		QString parseError; int line, column;
+		QFile hFile(hFilePath);
+		if (!hFile.open(QIODevice::ReadOnly) ||
+		        !hDoc.setContent(&hFile, &parseError, &line, &column))
+		{
+			qWarning() << "Error parsing:" << hFilePath
+			           << parseError << line << column;
+			continue;
+		}
+
+		QFileInfo hfi(hFile);
+		QString headerFileName(hfi.fileName());
+		headerFileName.replace(QString("_8h.xml"), QString(".h"));
+
+		QDomNodeList sectiondefs = hDoc.elementsByTagName("sectiondef");
+		for(int j = 0; j < sectiondefs.size(); ++j)
+		{
+			QDomElement sectDef = sectiondefs.item(j).toElement();
+
+			if( sectDef.attributes().namedItem("kind").nodeValue() != "var"
+			        || sectDef.elementsByTagName("jsonapi").isEmpty() )
+				continue;
+
+			QString instanceName =
+			        sectDef.elementsByTagName("name").item(0).toElement().text();
+			QString typeName =
+			        sectDef.elementsByTagName("ref").item(0).toElement().text();
+			QString typeFilePath(doxPrefix);
+			typeFilePath += sectDef.elementsByTagName("ref").item(0)
+			        .attributes().namedItem("refid").nodeValue();
+			typeFilePath += ".xml";
+
+			QDomDocument typeDoc;
+			QFile typeFile(typeFilePath);
+			if ( !typeFile.open(QIODevice::ReadOnly) ||
+			     !typeDoc.setContent(&typeFile, &parseError, &line, &column) )
+			{
+				qWarning() << "Error parsing:" << typeFilePath
+				           << parseError << line << column;
+				continue;
+			}
+
+			QDomNodeList members = typeDoc.elementsByTagName("member");
+			for (int i = 0; i < members.size(); ++i)
+			{
+				QDomNode member = members.item(i);
+				QString refid(member.attributes().namedItem("refid").nodeValue());
+				QString methodName(member.firstChildElement("name").toElement().text());
+				QString wrapperName(instanceName+methodName+"Wrapper");
+
+				QDomDocument defDoc;
+				QString defFilePath(doxPrefix + refid.split('_')[0] + ".xml");
+				QFile defFile(defFilePath);
+				if ( !defFile.open(QIODevice::ReadOnly) ||
+				     !defDoc.setContent(&defFile, &parseError, &line, &column) )
+				{
+					qWarning() << "Error parsing:" << defFilePath
+					           << parseError << line << column;
+					continue;
+				}
+
+				QDomElement memberdef;
+				QDomNodeList memberdefs = typeDoc.elementsByTagName("memberdef");
+				for (int k = 0; k < memberdefs.size(); ++k)
+				{
+					QDomElement tmpMBD = memberdefs.item(k).toElement();
+					QString tmpId = tmpMBD.attributes().namedItem("id").nodeValue();
+					QString tmpKind = tmpMBD.attributes().namedItem("kind").nodeValue();
+					bool hasJsonApi = !tmpMBD.elementsByTagName("jsonapi").isEmpty();
+					if( tmpId == refid && tmpKind == "function" && hasJsonApi )
+					{
+						memberdef = tmpMBD;
+						break;
+					}
+				}
+
+				if(memberdef.isNull()) continue;
+
+				QString apiPath("/" + instanceName + "/" + methodName);
+				QString retvalType = memberdef.firstChildElement("type").text();
+				QMap<QString,MethodParam> paramsMap;
+				QStringList orderedParamNames;
+
+				QDomNodeList params = memberdef.elementsByTagName("param");
+				for (int k = 0; k < params.size(); ++k)
+				{
+					QDomElement tmpPE = params.item(k).toElement();
+					MethodParam tmpParam;
+					tmpParam.name = tmpPE.firstChildElement("declname").text();
+					QDomElement tmpType = tmpPE.firstChildElement("type");
+					QString& pType(tmpParam.type);
+					pType = tmpType.text();
+					if(pType.startsWith("const ")) pType.remove(0,6);
+					pType.replace(QString("&"), QString());
+					pType.replace(QString(" "), QString());
+					paramsMap.insert(tmpParam.name, tmpParam);
+					orderedParamNames.push_back(tmpParam.name);
+				}
+
+				QDomNodeList parameternames = memberdef.elementsByTagName("parametername");
+				for (int k = 0; k < parameternames.size(); ++k)
+				{
+					QDomElement tmpPN = parameternames.item(k).toElement();
+					MethodParam& tmpParam = paramsMap[tmpPN.text()];
+					QString tmpD = tmpPN.attributes().namedItem("direction").nodeValue();
+					tmpParam.in = tmpD.contains("in");
+					tmpParam.out = tmpD.contains("out");
+				}
+
+				qDebug() << instanceName << apiPath << retvalType << typeName
+				         << methodName;
+				for (const QString& pn : orderedParamNames)
+				{
+					const MethodParam& mp(paramsMap[pn]);
+					qDebug() << "\t" << mp.type << mp.name << mp.in << mp.out;
+				}
+
+				QString retvalSerialization;
+				if(retvalType != "void")
+					retvalSerialization = "\t\t\tRS_SERIAL_PROCESS(retval);";
+
+				QString paramsDeclaration;
+				QString inputParamsDeserialization;
+				QString outputParamsSerialization;
+				for (const QString& pn : orderedParamNames)
+				{
+					const MethodParam& mp(paramsMap[pn]);
+					paramsDeclaration += "\t\t" + mp.type + " " + mp.name + ";\n";
+					if(mp.in)
+						inputParamsDeserialization += "\t\t\tRS_SERIAL_PROCESS("
+						        + mp.name + ");\n";
+					if(mp.out)
+						outputParamsSerialization += "\t\t\tRS_SERIAL_PROCESS("
+						        + mp.name + ");\n";
+				}
+
+				QMap<QString,QString> substitutionsMap;
+				substitutionsMap.insert("instanceName", instanceName);
+				substitutionsMap.insert("methodName", methodName);
+				substitutionsMap.insert("paramsDeclaration", paramsDeclaration);
+				substitutionsMap.insert("inputParamsDeserialization", inputParamsDeserialization);
+				substitutionsMap.insert("outputParamsSerialization", outputParamsSerialization);
+				substitutionsMap.insert("retvalSerialization", retvalSerialization);
+				substitutionsMap.insert("retvalType", retvalType);
+				substitutionsMap.insert("callParamsList", orderedParamNames.join(", "));
+				substitutionsMap.insert("wrapperName", wrapperName);
+				substitutionsMap.insert("headerFileName", headerFileName);
+
+				QFile templFile(sourcePath + "/method-wrapper-template.cpp.tmpl");
+				templFile.open(QIODevice::ReadOnly);
+				QString wrapperDef(templFile.readAll());
+
+				QMap<QString,QString>::iterator it;
+				for ( it = substitutionsMap.begin();
+				      it != substitutionsMap.end(); ++it )
+					wrapperDef.replace(QString("$%"+it.key()+"%$"), QString(it.value()), Qt::CaseSensitive);
+
+				wrappersDefFile.write(wrapperDef.toLocal8Bit());
+
+				QString wrapperDecl("void " + instanceName + methodName + "Wrapper(const std::shared_ptr<rb::Session> session);\n");
+				wrappersDeclFile.write(wrapperDecl.toLocal8Bit());
+
+
+				QString wrapperReg("registerHandler(\""+apiPath+"\", "+wrapperName+");\n");
+				wrappersRegisterFile.write(wrapperReg.toLocal8Bit());
+			}
+		}
+	}
+
+	return 0;
+}
--- a/jsonapi-generator/src/jsonapi-generator.pro
+++ b/jsonapi-generator/src/jsonapi-generator.pro
@ -0,0 +1,6 @@
+TARGET = jsonapi-generator
+
+QT *= core xml
+QT -= gui
+
+SOURCES += jsonapi-generator.cpp
--- a/jsonapi-generator/src/method-wrapper-template.cpp.tmpl
+++ b/jsonapi-generator/src/method-wrapper-template.cpp.tmpl
@ -0,0 +1,79 @@
+/*
+ * RetroShare JSON API
+ * Copyright (C) 2018  Gioacchino Mazzurco <gio@eigenlab.org>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <sstream>
+#include <memory>
+#include <restbed>
+#include "retroshare/$%headerFileName%$"
+
+namespace rb = restbed;
+
+void $%wrapperName%$(const std::shared_ptr<rb::Session> session)
+{
+	size_t reqSize = session->get_request()->get_header("Content-Length", 0);
+	session->fetch( reqSize, [](
+	                const std::shared_ptr<rb::Session> session,
+	                const rb::Bytes& body )
+	{
+		RsGenericSerializer::SerializeContext cReq(
+		            nullptr, 0,
+		            RsGenericSerializer::SERIALIZATION_FLAG_YIELDING );
+		RsJson& jReq(cReq.mJson);
+		jReq.Parse(reinterpret_cast<const char*>(body.data()), body.size());
+
+		RsGenericSerializer::SerializeContext cAns;
+		RsJson& jAns(cAns.mJson);
+
+		// if caller specified caller_data put it back in the answhere
+		const char kcd[] = "caller_data";
+		if(jReq.HasMember(kcd))
+		    jAns.AddMember(kcd, jReq[kcd], jAns.GetAllocator());
+
+$%paramsDeclaration%$
+
+		// deserialize input parameters from JSON
+		{
+			RsGenericSerializer::SerializeContext& ctx(cReq);
+			RsGenericSerializer::SerializeJob j(RsGenericSerializer::FROM_JSON);
+$%inputParamsDeserialization%$
+		}
+
+		// call retroshare C++ API
+		$%retvalType%$ retval = $%instanceName%$->$%methodName%$($%callParamsList%$);
+
+		// serialize out parameters and return value to JSON
+		{
+			RsGenericSerializer::SerializeContext& ctx(cAns);
+			RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON);
+$%retvalSerialization%$
+$%outputParamsSerialization%$
+		}
+
+		// return them to the API caller
+		std::stringstream ss;
+		ss << jAns;
+		std::string&& ans(ss.str());
+		const std::multimap<std::string, std::string> headers
+		{
+			{ "Content-Type", "text/json" },
+			{ "Content-Length", std::to_string(ans.length()) }
+		};
+		session->close(rb::OK, ans, headers);
+	} );
+}
+