From e62b8472345d54affdb0cacf02d0e9b90d0f5577 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 13 Dec 2021 18:42:28 +0100 Subject: [PATCH] Integrate python3 JSON API generator into libretroshare Add more options to CMake: Support for JSON API Support for forum deep index Fix bitdht CMake project name General CMake files improvements --- jsonapi-generator/README.adoc | 449 +----------------- .../async-method-wrapper-template.cpp.tmpl | 81 +--- .../src/jsonapi-generator-doxygen.conf | 231 +-------- .../src/method-wrapper-template.cpp.tmpl | 51 +- libbitdht/CMakeLists.txt | 2 +- libretroshare/CMakeLists.txt | 226 +++++++-- libretroshare/src/CMakeLists.txt | 68 ++- libretroshare/src/jsonapi/README.adoc | 447 +++++++++++++++++ .../async-method-wrapper-template.cpp.tmpl | 80 ++++ .../jsonapi/jsonapi-generator-doxygen.conf | 230 +++++++++ .../src/jsonapi/jsonapi-generator.py | 365 ++++++++++++++ .../jsonapi/method-wrapper-template.cpp.tmpl | 50 ++ 12 files changed, 1444 insertions(+), 836 deletions(-) mode change 100644 => 120000 jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl mode change 100644 => 120000 jsonapi-generator/src/jsonapi-generator-doxygen.conf mode change 100644 => 120000 jsonapi-generator/src/method-wrapper-template.cpp.tmpl create mode 100644 libretroshare/src/jsonapi/README.adoc create mode 100644 libretroshare/src/jsonapi/async-method-wrapper-template.cpp.tmpl create mode 100644 libretroshare/src/jsonapi/jsonapi-generator-doxygen.conf create mode 100755 libretroshare/src/jsonapi/jsonapi-generator.py create mode 100644 libretroshare/src/jsonapi/method-wrapper-template.cpp.tmpl diff --git a/jsonapi-generator/README.adoc b/jsonapi-generator/README.adoc index 64284b85c..cea59a117 100644 --- a/jsonapi-generator/README.adoc +++ b/jsonapi-generator/README.adoc @@ -1,447 +1,10 @@ -// SPDX-FileCopyrightText: (C) 2004-2019 Retroshare Team +// SPDX-FileCopyrightText: (C) 2021 Retroshare Team // SPDX-License-Identifier: CC0-1.0 -RetroShare JSON API -=================== += Moved -:Cxx: C++ +JSON API generator is now part of `libretroshare` look under `src/jsonapi/` +directory in `libretroshare` repository. -== How to use RetroShare JSON API - -Look for methods marked with +@jsonapi+ doxygen custom command into -+libretroshare/src/retroshare+. The method path is composed by service instance -pointer name like +rsGxsChannels+ for +RsGxsChannels+, and the method name like -+createGroup+ and pass the input paramethers as a JSON object. - -.Service instance pointer in rsgxschannels.h -[source,cpp] --------------------------------------------------------------------------------- -/** - * Pointer to global instance of RsGxsChannels service implementation - * @jsonapi{development} - */ -extern RsGxsChannels* rsGxsChannels; --------------------------------------------------------------------------------- - -.Method declaration in rsgxschannels.h -[source,cpp] --------------------------------------------------------------------------------- - /** - * @brief Request channel creation. - * The action is performed asyncronously, so it could fail in a subsequent - * phase even after returning true. - * @jsonapi{development} - * @param[out] token Storage for RsTokenService token to track request - * status. - * @param[in] group Channel data (name, description...) - * @return false on error, true otherwise - */ - virtual bool createGroup(uint32_t& token, RsGxsChannelGroup& group) = 0; --------------------------------------------------------------------------------- - -.paramethers.json -[source,json] --------------------------------------------------------------------------------- -{ - "group":{ - "mMeta":{ - "mGroupName":"JSON test group", - "mGroupFlags":4, - "mSignFlags":520 - }, - "mDescription":"JSON test group description" - }, - "caller_data":"Here can go any kind of JSON data (even objects) that the caller want to get back together with the response" -} --------------------------------------------------------------------------------- - -.Calling the JSON API with curl on the terminal -[source,bash] --------------------------------------------------------------------------------- -curl -u $API_USER --data @paramethers.json http://127.0.0.1:9092/rsGxsChannels/createGroup --------------------------------------------------------------------------------- - -.JSON API call result -[source,json] --------------------------------------------------------------------------------- -{ - "caller_data": "Here can go any kind of JSON data (even objects) that the caller want to get back together with the response", - "retval": true, - "token": 3 -} --------------------------------------------------------------------------------- - -Even if it is less efficient because of URL encoding HTTP +GET+ method is -supported too, so in cases where the client cannot use +POST+ she can still use -+GET+ taking care of encoding the JSON data. With +curl+ this can be done at -least in two different ways. - -.Calling the JSON API with GET method with curl on the terminal -[source,bash] --------------------------------------------------------------------------------- -curl -u $API_USER --get --data-urlencode jsonData@paramethers.json \ - http://127.0.0.1:9092/rsGxsChannels/createGroup --------------------------------------------------------------------------------- - -Letting +curl+ do the encoding is much more elegant but it is semantically -equivalent to the following. - -.Calling the JSON API with GET method and pre-encoded data with curl on the terminal --------------------------------------------------------------------------------- -curl -u $API_USER http://127.0.0.1:9092/rsGxsChannels/createGroup?jsonData=%7B%0A%20%20%20%20%22group%22%3A%7B%0A%20%20%20%20%20%20%20%20%22mMeta%22%3A%7B%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mGroupName%22%3A%22JSON%20test%20group%22%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mGroupFlags%22%3A4%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mSignFlags%22%3A520%0A%20%20%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%20%20%22mDescription%22%3A%22JSON%20test%20group%20description%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22caller_data%22%3A%22Here%20can%20go%20any%20kind%20of%20JSON%20data%20%28even%20objects%29%20that%20the%20caller%20want%20to%20get%20back%20together%20with%20the%20response%22%0A%7D --------------------------------------------------------------------------------- - -Note that using +GET+ method +?jsonData=+ and then the JSON data URL encoded are -added after the path in the HTTP request. - - -== JSON API authentication - -Most of JSON API methods require authentication as they give access to -RetroShare user data, and we don't want any application running on the system -eventually by other users be able to access private data indiscriminately. -JSON API support HTTP Basic as authentication scheme, this is enough as JSON API -server is intented for usage on the same system (127.0.0.1) not over an -untrusted network. -If you need to use JSON API over an untrusted network consider using a reverse -proxy with HTTPS such as NGINX in front of JSON API server. -If RetroShare login has been effectuated through the JSON API you can use your -location SSLID as username and your PGP password as credential for the JSON API, -but we suggests you use specific meaningful and human readable credentials for -each JSON API client so the human user can have better control over which client -can access the JSON API. - -.NewToken.json -[source,json] --------------------------------------------------------------------------------- -{ - "token": "myNewUser:myNewPassword" -} --------------------------------------------------------------------------------- - -.An authenticated client can authorize new tokens like this --------------------------------------------------------------------------------- -curl -u $API_USER --data @NewToken.json http://127.0.0.1:9092/jsonApiServer/authorizeToken --------------------------------------------------------------------------------- - -.An unauthenticated JSON API client can request access with --------------------------------------------------------------------------------- -curl --data @NewToken.json http://127.0.0.1:9092/jsonApiServer/requestNewTokenAutorization --------------------------------------------------------------------------------- - -When an unauthenticated client request his token to be authorized, JSON API -server will try to ask confirmation to the human user if possible through -+mNewAccessRequestCallback+, if it is not possible or the user didn't authorized -the token +false+ is returned. - - -== Offer new RetroShare services through JSON API - -To offer a retroshare service through the JSON API, first of all one need find -the global pointer to the service instance and document it in doxygen syntax, -plus marking with the custom doxygen command +@jsonapi{RS_VERSION}+ where -+RS_VERSION+ is the retroshare version in which this service became available -with the current semantic (major changes to the service semantic, changes the -meaning of the service itself, so the version should be updated in the -documentation in that case). - -.Service instance pointer in rsgxschannels.h -[source,cpp] --------------------------------------------------------------------------------- -/** - * Pointer to global instance of RsGxsChannels service implementation - * @jsonapi{development} - */ -extern RsGxsChannels* rsGxsChannels; --------------------------------------------------------------------------------- - - -Once the service instance itself is known to the JSON API you need to document -in doxygen syntax and mark with the custom doxygen command -+@jsonapi{RS_VERSION}+ the methods of the service that you want to make -available through JSON API. - -.Offering RsGxsChannels::getChannelDownloadDirectory in rsgxschannels.h -[source,cpp] --------------------------------------------------------------------------------- - /** - * Get download directory for the given channel - * @jsonapi{development} - * @param[in] channelId id of the channel - * @param[out] directory reference to string where to store the path - * @return false on error, true otherwise - */ - virtual bool getChannelDownloadDirectory( const RsGxsGroupId& channelId, - std::string& directory ) = 0; --------------------------------------------------------------------------------- - -For each paramether you must specify if it is used as input +@param[in]+ as -output +@param[out]+ or both +@param[inout]+. Paramethers and return value -types must be of a type supported by +RsTypeSerializer+ which already support -most basic types (+bool+, +std::string+...), +RsSerializable+ and containers of -them like +std::vector+. Paramethers passed by value and by -reference of those types are both supported, while passing by pointer is not -supported. If your paramether or return +class+/+struct+ type is not supported -yet by +RsTypeSerializer+ most convenient approach is to make it derive from -+RsSerializable+ and implement +serial_process+ method like I did with -+RsGxsChannelGroup+. - -.Deriving RsGxsChannelGroup from RsSerializable in rsgxschannels.h -[source,cpp] --------------------------------------------------------------------------------- -struct RsGxsChannelGroup : RsSerializable -{ - RsGroupMetaData mMeta; - std::string mDescription; - RsGxsImage mImage; - - bool mAutoDownload; - - /// @see RsSerializable - virtual void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx ) - { - RS_SERIAL_PROCESS(mMeta); - RS_SERIAL_PROCESS(mDescription); - RS_SERIAL_PROCESS(mImage); - RS_SERIAL_PROCESS(mAutoDownload); - } -}; --------------------------------------------------------------------------------- - -You can do the same recursively for any member of your +struct+ that is not yet -supported by +RsTypeSerializer+. - -Some Retroshare {Cxx} API functions are asyncronous, historically RetroShare -didn't follow a policy on how to expose asyncronous API so differents services -and some times even differents method of the same service follow differents -asyncronous patterns, thus making automatic generation of JSON API wrappers for -those methods impractical. Instead of dealing with all those differents patterns -I have chosed to support only one new pattern taking advantage of modern {Cxx}11 -and restbed features. On the {Cxx}11 side lambdas and +std::function+s are used, -on the restbed side Server Side Events are used to send asyncronous results. - -Lets see an example so it will be much esier to understand. - -.RsGxsChannels::turtleSearchRequest asyncronous API -[source,cpp] --------------------------------------------------------------------------------- - /** - * @brief Request remote channels search - * @jsonapi{development} - * @param[in] matchString string to look for in the search - * @param multiCallback function that will be called each time a search - * result is received - * @param[in] maxWait maximum wait time in seconds for search results - * @return false on error, true otherwise - */ - virtual bool turtleSearchRequest( - const std::string& matchString, - const std::function& multiCallback, - std::time_t maxWait = 300 ) = 0; --------------------------------------------------------------------------------- - -+RsGxsChannels::turtleSearchRequest(...)+ is an asyncronous method because it -send a channel search request on turtle network and then everytime a result is -received from the network +multiCallback+ is called and the result is passed as -parameter. To be supported by the automatic JSON API wrappers generator an -asyncronous method need a parameter of type +std::function+ called -+callback+ if the callback will be called only once or +multiCallback+ if the -callback is expected to be called more then once like in this case. -A second mandatory parameter is +maxWait+ of type +std::time_t+ it indicates the -maximum amount of time in seconds for which the caller is willing to wait for -results, in case the timeout is reached the callback will not be called anymore. - -[IMPORTANT] -================================================================================ -+callback+ and +multiCallback+ parameters documentation must *not* specify -+[in]+, +[out]+, +[inout]+, in Doxygen documentation as this would fool the -automatic wrapper generator, and ultimately break the compilation. -================================================================================ - -.RsFiles::turtleSearchRequest asyncronous JSON API usage example -[source,bash] --------------------------------------------------------------------------------- -$ cat turtle_search.json -{ - "matchString":"linux" -} -$ curl --data @turtle_search.json http://127.0.0.1:9092/rsFiles/turtleSearchRequest -data: {"retval":true} - -data: {"results":[{"size":157631,"hash":"69709b4d01025584a8def5cd78ebbd1a3cf3fd05","name":"kill_bill_linux_1024x768.jpg"},{"size":192560,"hash":"000000000000000000009a93e5be8486c496f46c","name":"coffee_box_linux2.jpg"},{"size":455087,"hash":"9a93e5be8486c496f46c00000000000000000000","name":"Linux.png"},{"size":182004,"hash":"e8845280912ebf3779e400000000000000000000","name":"Linux_2_6.png"}]} - -data: {"results":[{"size":668,"hash":"e8845280912ebf3779e400000000000000000000","name":"linux.png"},{"size":70,"hash":"e8845280912ebf3779e400000000000000000000","name":"kali-linux-2016.2-amd64.txt.sha1sum"},{"size":3076767744,"hash":"e8845280912ebf3779e400000000000000000000","name":"kali-linux-2016.2-amd64.iso"},{"size":2780872,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.bin"},{"size":917504,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.lzma"},{"size":2278404096,"hash":"e8845280912ebf3779e400000000000000000000","name":"gentoo-linux-livedvd-amd64-multilib-20160704.iso"},{"size":151770333,"hash":"e8845280912ebf3779e400000000000000000000","name":"flashtool-0.9.23.0-linux.tar.7z"},{"size":2847372,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.elf"},{"size":1310720,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.gz"},{"size":987809,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux-lzma.elf"}]} - --------------------------------------------------------------------------------- - -By default JSON API methods requires client authentication and their wrappers -are automatically generated by +json-api-generator+. -In some cases methods need do be accessible without authentication such as -+rsLoginHelper/getLocations+ so in the doxygen documentaion they have the custom -command +@jsonapi{RS_VERSION,unauthenticated}+. -Other methods such as +/rsControl/rsGlobalShutDown+ need special care so they -are marked with the custom doxygen command +@jsonapi{RS_VERSION,manualwrapper}+ -and their wrappers are not automatically generated but written manually into -+JsonApiServer::JsonApiServer(...)+. - -== Quirks - -=== 64 bits integers handling - -While JSON doesn't have problems representing 64 bits integers JavaScript, Dart -and other languages are not capable to handle those numbers natively. -To overcome this limitation JSON API output 64 bit integers as an object with -two keys, one as proper integer and one as string representation. - -.JSON API 64 bit integer output example -[source,json] --------------------------------------------------------------------------------- -"lobby_id": { "xint64": 6215642878098695544, "xstr64": "6215642878098695544" } --------------------------------------------------------------------------------- - -So from languages that have proper 64bit integers support like Python or C++ one -better read from `xint64` which is represented as a JSON integer, from languages -where there is no proper 64bit integers support like JavaScript one can read from -`xstr64` which is represented as JSON string (note that the first is not wrapped -in "" while the latter is). - -When one input a 64bit integer into the JSON API it first try to parse it as if -it was sent the old way for retrocompatibility. - -.JSON API 64 bit integer deprecated format input example -[source,json] --------------------------------------------------------------------------------- -"lobby_id":6215642878098695544 --------------------------------------------------------------------------------- - -This way is *DEPRECATED* and may disappear in the future, it is TEMPORALLY kept -only for retrocompatibiliy with old clients. - -If retrocompatible parsing attempt fail then it try to parse with the new way -with proper JSON integer format. - -.JSON API 64 bit integer new proper integer format input example -[source,json] --------------------------------------------------------------------------------- -lobby_id": { "xint64": 6215642878098695544 } --------------------------------------------------------------------------------- - -If this fails then it try to parse with the new way with JSON string format. - -.JSON API 64 bit integer new string format input example -[source,json] --------------------------------------------------------------------------------- -"lobby_id": { "xstr64": "6215642878098695544" } --------------------------------------------------------------------------------- - -[WARNING] -================================================================================ -Clients written in languages without proper 64bit integers support must -use *ONLY* the string format otherwise they will send approximated values and -get unexpected results from the JSON API, because parsing will success but the -value will not be exactly the one you believe you sent. -================================================================================ - - -== A bit of history - -=== First writings about this - -The previous attempt of exposing a RetroShare JSON API is called +libresapi+ and -unfortunatley it requires a bunch of boilerplate code when we want to expose -something present in the {Cxx} API in the JSON API. - -As an example here you can see the libresapi that exposes part of the retroshare -chat {Cxx} API and lot of boilerplate code just to convert {Cxx} objects to JSON - -https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ChatHandler.cpp#L44 - -To avoid the {Cxx} to JSON and back conversion boilerplate code I have worked out -an extension to our {Cxx} serialization code so it is capable to serialize and -deserialize to JSON you can see it in this pull request - -https://github.com/RetroShare/RetroShare/pull/1155 - -So first step toward having a good API is to take advantage of the fact that RS -is now capable of converting C++ objects from and to JSON. - -The current API is accessible via HTTP and unix socket, there is no -authentication in both of them, so anyone having access to the HTTP server or to -the unix socket can access the API without extra restrictions. -Expecially for the HTTP API this is a big risk because also if the http server -listen on 127.0.0.1 every application on the machine (even rogue javascript -running on your web browser) can access that and for example on android it is -not safe at all (because of that I implemented the unix socket access so at -least in android API was reasonably safe) because of this. - -A second step to improve the API would be to implement some kind of API -authentication mechanism (it would be nice that the mechanism is handled at API -level and not at transport level so we can use it for any API trasport not just -HTTP for example) - -The HTTP server used by libresapi is libmicrohttpd server that is very minimal, -it doesn't provide HTTPS nor modern HTTP goodies, like server notifications, -websockets etc. because the lack of support we have a token polling mechanism in -libresapi to avoid polling for every thing but it is still ugly, so if we can -completely get rid of polling in the API that would be really nice. -I have done a crawl to look for a replacement and briefly looked at - -- https://www.gnu.org/software/libmicrohttpd/ -- http://wolkykim.github.io/libasyncd/ -- https://github.com/corvusoft/restbed -- https://code.facebook.com/posts/1503205539947302/introducing-proxygen-facebook-s-c-http-framework/ -- https://github.com/cmouse/yahttp - -taking in account a few metrics like modern HTTP goodies support, license, -platform support, external dependencies and documentation it seemed to me that -restbed is the more appropriate. - -Another source of boilerplate code into libresapi is the mapping between JSON -API requests and C++ API methods as an example you can look at this - -https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ChatHandler.cpp#L158 - -and this - -https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ApiServer.cpp#L253 - -The abstract logic of this thing is, when libreasapi get a request like -+/chat/initiate_distant_chat+ then call -+ChatHandler::handleInitiateDistantChatConnexion+ which in turn is just a -wrapper of +RsMsgs::initiateDistantChatConnexion+ all this process is basically -implemented as boilerplate code and would be unnecessary in a smarter design of -the API because almost all the information needed is already present in the -C++ API +libretroshare/src/retroshare+. - -So a third step to improve the JSON API would be to remove this source of -boilerplate code by automatizing the mapping between C++ and JSON API call. - -This may result a little tricky as language parsing or other adevanced things -may be required. - -Hope this dive is useful for you + -Cheers + -G10h4ck - -=== Second writings about this - -I have been investigating a bit more about: -[verse, G10h4ck] -________________________________________________________________________________ -So a third step to improve the JSON API would be to remove this source of -boilerplate code by automatizing the mapping between C++ and JSON API call -________________________________________________________________________________ - -After spending some hours investigating this topic the most reasonable approach -seems to: - -1. Properly document headers in +libretroshare/src/retroshare/+ in doxygen syntax -specifying wihich params are input and/or output (doxygen sysntax for this is -+@param[in/out/inout]+) this will be the API documentation too. - -2. At compile time use doxygen to generate XML description of the headers and use -the XML to generate the JSON api server stub. -http://www.stack.nl/~dimitri/doxygen/manual/customize.html#xmlgenerator - -3. Enjoy +This directory and all it's content is kept as is only for retro-compatibility +with the old, deprecated `qmake` build system. diff --git a/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl b/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl deleted file mode 100644 index 29f53ad48..000000000 --- a/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl +++ /dev/null @@ -1,80 +0,0 @@ -/******************************************************************************* - * RetroShare JSON API * - * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU Affero General Public License version 3 as * - * published by the Free Software Foundation. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Affero General Public License for more details. * - * * - * You should have received a copy of the GNU Affero General Public License * - * along with this program. If not, see . * - * * - *******************************************************************************/ - -registerHandler( "$%apiPath%$", - [this](const std::shared_ptr session) -{ - const std::multimap headers - { - { "Connection", "keep-alive" }, - { "Content-Type", "text/event-stream" } - }; - session->yield(rb::OK, headers); - - size_t reqSize = session->get_request()->get_header("Content-Length", 0); - session->fetch( reqSize, [this]( - const std::shared_ptr session, - const rb::Bytes& body ) - { - INITIALIZE_API_CALL_JSON_CONTEXT; - - if( !checkRsServicePtrReady( - $%instanceName%$, "$%instanceName%$", cAns, session ) ) - return; - -$%paramsDeclaration%$ - -$%inputParamsDeserialization%$ - - const std::weak_ptr weakService(mService); - const std::weak_ptr weakSession(session); - $%callbackName%$ = [weakService, weakSession]($%callbackParams%$) - { - auto session = weakSession.lock(); - if(!session || session->is_closed()) return; - - auto lService = weakService.lock(); - if(!lService || lService->is_down()) return; - -$%callbackParamsSerialization%$ - - std::stringstream sStream; - sStream << "data: " << compactJSON << ctx.mJson << "\n\n"; - const std::string message = sStream.str(); - - lService->schedule( [weakSession, message]() - { - auto session = weakSession.lock(); - if(!session || session->is_closed()) return; - session->yield(message); - $%sessionEarlyClose%$ - } ); - }; - -$%functionCall%$ - -$%outputParamsSerialization%$ - - // return them to the API caller - std::stringstream message; - message << "data: " << compactJSON << cAns.mJson << "\n\n"; - session->yield(message.str()); - $%sessionDelayedClose%$ - } ); -}, $%requiresAuth%$ ); diff --git a/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl b/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl new file mode 120000 index 000000000..1ae671997 --- /dev/null +++ b/jsonapi-generator/src/async-method-wrapper-template.cpp.tmpl @@ -0,0 +1 @@ +../../libretroshare/src/jsonapi/async-method-wrapper-template.cpp.tmpl \ No newline at end of file diff --git a/jsonapi-generator/src/jsonapi-generator-doxygen.conf b/jsonapi-generator/src/jsonapi-generator-doxygen.conf deleted file mode 100644 index 0edd1de3e..000000000 --- a/jsonapi-generator/src/jsonapi-generator-doxygen.conf +++ /dev/null @@ -1,230 +0,0 @@ -DOXYFILE_ENCODING = UTF-8 -PROJECT_NAME = "libretroshare" - -ALIASES += jsonapi{1}="\xmlonly\endxmlonly" -ALIASES += jsonapi{2}="\xmlonly\endxmlonly" - -# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the -# documentation from any documented member that it re-implements. -# The default value is: YES. - -INHERIT_DOCS = YES - -# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments -# according to the Markdown format, which allows for more readable -# documentation. See http://daringfireball.net/projects/markdown/ for details. -# The output of markdown processing is further processed by doxygen, so you can -# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in -# case of backward compatibilities issues. -# The default value is: YES. - -MARKDOWN_SUPPORT = YES - -# When enabled doxygen tries to link words that correspond to documented -# classes, or namespaces to their corresponding documentation. Such a link can -# be prevented in individual cases by putting a % sign in front of the word or -# globally by setting AUTOLINK_SUPPORT to NO. -# The default value is: YES. - -AUTOLINK_SUPPORT = YES - - -# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in -# documentation are documented, even if no documentation was available. Private -# class members and static file members will be hidden unless the -# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. -# Note: This will also disable the warnings about undocumented members that are -# normally produced when WARNINGS is set to YES. -# The default value is: NO. - -EXTRACT_ALL = YES - - -# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all -# undocumented members inside documented classes or files. If set to NO these -# members will be included in the various overviews, but no documentation -# section is generated. This option has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_MEMBERS = NO - -# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all -# undocumented classes that are normally visible in the class hierarchy. If set -# to NO, these classes will be included in the various overviews. This option -# has no effect if EXTRACT_ALL is enabled. -# The default value is: NO. - -HIDE_UNDOC_CLASSES = NO - - -#--------------------------------------------------------------------------- -# Configuration options related to the input files -#--------------------------------------------------------------------------- - -# The INPUT tag is used to specify the files and/or directories that contain -# documented source files. You may enter file names like myfile.cpp or -# directories like /usr/src/myproject. Separate the files or directories with -# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING -# Note: If this tag is empty the current directory is searched. - -#INPUT = - -# This tag can be used to specify the character encoding of the source files -# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses -# libiconv (or the iconv built into libc) for the transcoding. See the libiconv -# documentation (see: https://www.gnu.org/software/libiconv/) for the list of -# possible encodings. -# The default value is: UTF-8. - -INPUT_ENCODING = UTF-8 - -# If the value of the INPUT tag contains directories, you can use the -# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and -# *.h) to filter out the source-files in the directories. -# -# Note that for custom extensions or not directly supported extensions you also -# need to set EXTENSION_MAPPING for the extension otherwise the files are not -# read by doxygen. -# -# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, -# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, -# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, -# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, -# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. - -FILE_PATTERNS = *.c \ - *.cc \ - *.cxx \ - *.cpp \ - *.c++ \ - *.java \ - *.ii \ - *.ixx \ - *.ipp \ - *.i++ \ - *.inl \ - *.idl \ - *.ddl \ - *.odl \ - *.h \ - *.hh \ - *.hxx \ - *.hpp \ - *.h++ \ - *.cs \ - *.d \ - *.php \ - *.php4 \ - *.php5 \ - *.phtml \ - *.inc \ - *.m \ - *.markdown \ - *.md \ - *.mm \ - *.dox \ - *.py \ - *.pyw \ - *.f90 \ - *.f95 \ - *.f03 \ - *.f08 \ - *.f \ - *.for \ - *.tcl \ - *.vhd \ - *.vhdl \ - *.ucf \ - *.qsf - -# The RECURSIVE tag can be used to specify whether or not subdirectories should -# be searched for input files as well. -# The default value is: NO. - -RECURSIVE = YES - -# The EXCLUDE tag can be used to specify files and/or directories that should be -# excluded from the INPUT source files. This way you can easily exclude a -# subdirectory from a directory tree whose root is specified with the INPUT tag. -# -# Note that relative paths are relative to the directory from which doxygen is -# run. - -EXCLUDE = - -# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or -# directories that are symbolic links (a Unix file system feature) are excluded -# from the input. -# The default value is: NO. - -EXCLUDE_SYMLINKS = NO - -# If the value of the INPUT tag contains directories, you can use the -# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude -# certain files from those directories. -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories for example use the pattern */test/* - -EXCLUDE_PATTERNS = - -# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names -# (namespaces, classes, functions, etc.) that should be excluded from the -# output. The symbol name can be a fully qualified name, a word, or if the -# wildcard * is used, a substring. Examples: ANamespace, AClass, -# AClass::ANamespace, ANamespace::*Test -# -# Note that the wildcards are matched against the file with absolute path, so to -# exclude all test directories use the pattern */test/* - -EXCLUDE_SYMBOLS = - - -#--------------------------------------------------------------------------- -# Configuration options related to the HTML output -#--------------------------------------------------------------------------- - -# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output -# The default value is: YES. - -GENERATE_HTML = NO - -# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. -# The default value is: YES. - -GENERATE_LATEX = NO - -# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that -# captures the structure of the code including all documentation. -# The default value is: NO. - -GENERATE_XML = YES - - -#--------------------------------------------------------------------------- -# Configuration options related to the preprocessor -#--------------------------------------------------------------------------- - -# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all -# C-preprocessor directives found in the sources and include files. -# The default value is: YES. - -ENABLE_PREPROCESSING = YES - -# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names -# in the source code. If set to NO, only conditional compilation will be -# performed. Macro expansion can be done in a controlled way by setting -# EXPAND_ONLY_PREDEF to YES. -# The default value is: NO. -# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. - -MACRO_EXPANSION = NO - -# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then -# the macro expansion is limited to the macros specified with the PREDEFINED and -# EXPAND_AS_DEFINED tags. -# The default value is: NO. -# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. - -EXPAND_ONLY_PREDEF = NO - diff --git a/jsonapi-generator/src/jsonapi-generator-doxygen.conf b/jsonapi-generator/src/jsonapi-generator-doxygen.conf new file mode 120000 index 000000000..9aff33c39 --- /dev/null +++ b/jsonapi-generator/src/jsonapi-generator-doxygen.conf @@ -0,0 +1 @@ +../../libretroshare/src/jsonapi/jsonapi-generator-doxygen.conf \ No newline at end of file diff --git a/jsonapi-generator/src/method-wrapper-template.cpp.tmpl b/jsonapi-generator/src/method-wrapper-template.cpp.tmpl deleted file mode 100644 index a3927fba9..000000000 --- a/jsonapi-generator/src/method-wrapper-template.cpp.tmpl +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * RetroShare JSON API * - * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU Affero General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Affero General Public License for more details. * - * * - * You should have received a copy of the GNU Affero General Public License * - * along with this program. If not, see . * - * * - *******************************************************************************/ - -registerHandler( "$%apiPath%$", - [](const std::shared_ptr session) -{ - size_t reqSize = session->get_request()->get_header("Content-Length", 0); - session->fetch( reqSize, []( - const std::shared_ptr session, - const rb::Bytes& body ) - { - INITIALIZE_API_CALL_JSON_CONTEXT; - - if( !checkRsServicePtrReady( - $%instanceName%$, "$%instanceName%$", cAns, session ) ) - return; - -$%paramsDeclaration%$ - - // deserialize input parameters from JSON -$%inputParamsDeserialization%$ - - // call retroshare C++ API -$%functionCall%$ - - // serialize out parameters and return value to JSON -$%outputParamsSerialization%$ - - // return them to the API caller - DEFAULT_API_CALL_JSON_RETURN(rb::OK); - } ); -}, $%requiresAuth%$ ); - diff --git a/jsonapi-generator/src/method-wrapper-template.cpp.tmpl b/jsonapi-generator/src/method-wrapper-template.cpp.tmpl new file mode 120000 index 000000000..717044ed2 --- /dev/null +++ b/jsonapi-generator/src/method-wrapper-template.cpp.tmpl @@ -0,0 +1 @@ +../../libretroshare/src/jsonapi/method-wrapper-template.cpp.tmpl \ No newline at end of file diff --git a/libbitdht/CMakeLists.txt b/libbitdht/CMakeLists.txt index 2a996de48..af352651c 100644 --- a/libbitdht/CMakeLists.txt +++ b/libbitdht/CMakeLists.txt @@ -6,7 +6,7 @@ # SPDX-License-Identifier: CC0-1.0 cmake_minimum_required (VERSION 2.8.12) -project(libbitdht) +project(bitdht) file( GLOB BITDHT_SOURCES diff --git a/libretroshare/CMakeLists.txt b/libretroshare/CMakeLists.txt index 0eb111dd7..93c8b04a5 100644 --- a/libretroshare/CMakeLists.txt +++ b/libretroshare/CMakeLists.txt @@ -8,6 +8,11 @@ cmake_minimum_required (VERSION 3.18.0) project(retroshare) +include(CMakeDependentOption) + +set(FETCHCONTENT_QUIET OFF) +include(FetchContent) + # sqlcipher option( RS_SQLCIPHER @@ -28,18 +33,22 @@ option( ON ) # use_dht_stunner -option( +cmake_dependent_option( RS_BITDHT_STUNNER "Use bitdht (BitTorrent DHT own implementation) for NAT type discovery and \ attempt the STUN (Session Traversal Utilities for NAT)" - ON ) + ON + "RS_BITDHT" + OFF ) # use_dht_stunner_ext_ip -option( +cmake_dependent_option( RS_BITDHT_STUNNER_EXT_IP "Use bitdht (BitTorrent DHT own implementation) stunner to figure out our \ external IP. As this purely relying on random DHT peers that answer our \ request, it can easily be abused. Therefore, it is turned off by default." + OFF + "RS_BITDHT_STUNNER" OFF ) # rs_jsonapi @@ -71,53 +80,45 @@ option( option( RS_MINIUPNPC "Forward ports in NAT router via miniupnpc" - ON -) + ON ) -include(CMakeDependentOption) cmake_dependent_option( RS_LIBUPNP "Forward ports in NAT router via libupnp (unstable)" OFF "NOT RS_MINIUPNPC" - OFF -) + OFF ) option( RS_LIBRETROSHARE_STATIC "Build RetroShare static library" - ON -) + ON ) cmake_dependent_option( RS_LIBRETROSHARE_SHARED "Build RetroShare shared library" OFF "NOT RS_LIBRETROSHARE_STATIC" - OFF -) + OFF ) # rs_deprecatedwarning option( RS_WARN_DEPRECATED "Print warning about RetroShare deprecated components usage during build" - ON -) + ON ) # rs_cppwarning option( RS_WARN_LESS "Silence a few at the moment very common warnings about RetroShare \ components during build" - OFF -) + OFF ) # rs_v07_changes option( RS_V07_BREAKING_CHANGES "Enable retro-compatibility breaking changes planned for RetroShare 0.7.0" - OFF -) + OFF ) set( RS_DATA_DIR @@ -127,8 +128,24 @@ set( ################################################################################ +find_package(Git REQUIRED) + +#function(check_submodule sPath) +# if(NOT EXISTS "${sPath}/.git" ) +# message("Initializing submodule ${sPath}") +# execute_process( +# COMMAND "${GIT_EXECUTABLE}" submodule update --init +# WORKING_DIRECTORY "${sPath}" +# COMMAND_ECHO STDERR +# COMMAND_ERROR_IS_FATAL ANY) +# endif() +#endfunction() + +################################################################################ + include(src/CMakeLists.txt) list(TRANSFORM RS_SOURCES PREPEND src/) +list(TRANSFORM RS_PUBLIC_HEADERS PREPEND src/) if(RS_LIBRETROSHARE_STATIC) add_library(${PROJECT_NAME} STATIC ${RS_SOURCES}) @@ -153,15 +170,150 @@ target_link_libraries(${PROJECT_NAME} PRIVATE OpenSSL::SSL OpenSSL::Crypto) ################################################################################ -add_subdirectory(../openpgpsdk ${CMAKE_BINARY_DIR}/openpgpsdk) +set(OPENPGPSDK_DEVEL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../openpgpsdk/") +if(EXISTS "${OPENPGPSDK_DEVEL_DIR}/.git" ) + message( + CHECK_PASS + "openpgpsdk submodule found at ${OPENPGPSDK_DEVEL_DIR} using it" ) + add_subdirectory(${OPENPGPSDK_DEVEL_DIR} ${CMAKE_BINARY_DIR}/openpgpsdk) +else() + FetchContent_Declare( + openpgpsdk + GIT_REPOSITORY "https://gitlab.com/RetroShare/openpgpsdk.git" + GIT_TAG "origin/master" + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE + TIMEOUT 10 + ) + FetchContent_MakeAvailable(openpgpsdk) +endif() + target_link_libraries(${PROJECT_NAME} PRIVATE openpgpsdk) +################################################################################ + if(RS_BITDHT) + set(BITDHT_DEVEL_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../libbitdht/") + if(EXISTS "${BITDHT_DEVEL_DIR}/.git" ) + message( + CHECK_PASS + "BitDHT submodule found at ${BITDHT_DEVEL_DIR} using it" ) + add_subdirectory(${BITDHT_DEVEL_DIR} ${CMAKE_BINARY_DIR}/bitdht) + else() + FetchContent_Declare( + bitdht + GIT_REPOSITORY "https://gitlab.com/RetroShare/bitdht.git" + GIT_TAG "origin/master" + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE + TIMEOUT 10 + ) + FetchContent_MakeAvailable(bitdht) + endif() + add_compile_definitions(RS_USE_BITDHT) - add_subdirectory(../libbitdht ${CMAKE_BINARY_DIR}/libbitdht) - target_link_libraries(${PROJECT_NAME} PRIVATE libbitdht) + target_link_libraries(${PROJECT_NAME} PRIVATE bitdht) + + if(RS_BITDHT_STUNNER) + add_compile_definitions(RS_USE_DHT_STUNNER) + + if(RS_BITDHT_STUNNER_EXT_IP) + # TODO: Refactor this define to use proper naming + add_compile_definitions(ALLOW_DHT_STUNNER) + endif(RS_BITDHT_STUNNER_EXT_IP) + endif(RS_BITDHT_STUNNER) endif(RS_BITDHT) +################################################################################ + +if(RS_JSON_API) + find_package(Doxygen REQUIRED) + find_package(Python3 REQUIRED) + + ## TODO: execute at build time instead that at cofiguration time see + ## add_custom_command or add_custom_target + + set( + JSON_API_GENERATOR_WORK_DIR + "${CMAKE_BINARY_DIR}/jsonapi-generator.workdir/" ) + + set( + JSON_API_GENERATOR_DOXYFILE + "${JSON_API_GENERATOR_WORK_DIR}/jsonapi-generator-doxygen.conf" ) + + set( + JSONAPI_GENERATOR_OUTPUT_DIR + "${JSON_API_GENERATOR_WORK_DIR}/src/" ) + + set( + JSONAPI_GENERATOR_SOURCE_DIR + "${CMAKE_CURRENT_SOURCE_DIR}/src/jsonapi/" ) + + set( + JSONAPI_GENERATOR_EXECUTABLE + "${JSONAPI_GENERATOR_SOURCE_DIR}/jsonapi-generator.py" ) + + file( + COPY "src/jsonapi/jsonapi-generator-doxygen.conf" + DESTINATION "${JSON_API_GENERATOR_WORK_DIR}" ) + + file( + APPEND + "${JSON_API_GENERATOR_DOXYFILE}" + "OUTPUT_DIRECTORY=${JSONAPI_GENERATOR_OUTPUT_DIR}\n" + "INPUT=${CMAKE_CURRENT_SOURCE_DIR}" ) + + add_custom_command( + OUTPUT + "${JSONAPI_GENERATOR_OUTPUT_DIR}/jsonapi-includes.inl" + "${JSONAPI_GENERATOR_OUTPUT_DIR}/jsonapi-wrappers.inl" + COMMAND ${DOXYGEN_EXECUTABLE} ${JSON_API_GENERATOR_DOXYFILE} + COMMAND + ${Python3_EXECUTABLE} ${JSONAPI_GENERATOR_EXECUTABLE} + ${JSONAPI_GENERATOR_SOURCE_DIR} ${JSONAPI_GENERATOR_OUTPUT_DIR} + MAIN_DEPENDENCY "${JSONAPI_GENERATOR_EXECUTABLE}" + DEPENDS ${JSON_API_GENERATOR_DOXYFILE} ${RS_PUBLIC_HEADERS} ) + + target_sources( + ${PROJECT_NAME} PRIVATE + "${JSONAPI_GENERATOR_OUTPUT_DIR}/jsonapi-includes.inl" + "${JSONAPI_GENERATOR_OUTPUT_DIR}/jsonapi-wrappers.inl" ) + + include_directories(${JSONAPI_GENERATOR_OUTPUT_DIR}) + + set(BUILD_TESTS OFF CACHE BOOL "Do not build restbed tests") + set(BUILD_SSL OFF CACHE BOOL "Do not build restbed SSL support") + + FetchContent_Declare( + restbed + GIT_REPOSITORY "https://github.com/Corvusoft/restbed.git" + GIT_TAG "4.8" + GIT_SUBMODULES dependency/asio dependency/catch + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE + TIMEOUT 10 + ) + FetchContent_MakeAvailable(restbed) + + target_link_libraries(${PROJECT_NAME} PRIVATE restbed) + + ## TODO: work around target_include_directories should be added upstream + include_directories(${restbed_SOURCE_DIR}/source/) + + add_compile_definitions(RS_JSONAPI) +endif(RS_JSON_API) + +################################################################################ + +if(RS_FORUM_DEEP_INDEX) + find_package(Xapian REQUIRED) + target_link_libraries(${PROJECT_NAME} PRIVATE ${XAPIAN_LIBRARIES}) + + add_compile_definitions(RS_DEEP_FORUMS_INDEX) +endif(RS_FORUM_DEEP_INDEX) + +################################################################################ + ## TODO: Check if https://github.com/rbock/sqlpp11 or ## https://github.com/rbock/sqlpp17 may improve GXS code if(RS_SQLCIPHER) @@ -196,12 +348,17 @@ if(RS_V07_BREAKING_CHANGES) V07_NON_BACKWARD_COMPATIBLE_CHANGE_UNNAMED ) endif() +if(RS_DH_PRIME_INIT_CHECK) + add_compile_definitions(RS_DISABLE_DIFFIE_HELLMAN_INIT_CHECK) +endif(RS_DH_PRIME_INIT_CHECK) + if(RS_MINIUPNPC) add_compile_definitions(RS_USE_LIBMINIUPNPC) endif(RS_MINIUPNPC) if(RS_LIBUPNP) - add_compile_definitions(RS_USE_LIBUPNP) + message(FATAL_ERROR "UPnP support via libupnp is currently not supported") + #add_compile_definitions(RS_USE_LIBUPNP) endif(RS_LIBUPNP) if(RS_GXS_SEND_ALL) @@ -209,13 +366,24 @@ if(RS_GXS_SEND_ALL) endif(RS_GXS_SEND_ALL) if(RS_BRODCAST_DISCOVERY) - add_subdirectory( - ../supportlibs/udp-discovery-cpp/ - ${CMAKE_BINARY_DIR}/supportlibs/udp-discovery-cpp/ ) - target_link_libraries(${PROJECT_NAME} PRIVATE udp-discovery) + ## TODO: upstream option to disable tests building + set(BUILD_EXAMPLE OFF CACHE BOOL "Do not build udp-discovery-cpp examples") + set(BUILD_TOOL OFF CACHE BOOL "Do not build udp-discovery-tool application") + FetchContent_Declare( + udp-discovery-cpp + GIT_REPOSITORY "https://github.com/truvorskameikin/udp-discovery-cpp.git" + GIT_TAG "origin/master" + GIT_SHALLOW TRUE + GIT_PROGRESS TRUE + TIMEOUT 10 + ) + FetchContent_MakeAvailable(udp-discovery-cpp) - ## Temporary work around target_include_directories should be added upstream - include_directories(../supportlibs/udp-discovery-cpp/) + target_link_libraries(${PROJECT_NAME} PRIVATE udp-discovery-cpp) + + ## TODO: Temporary work around target_include_directories should be added + ## upstream + include_directories(${udp-discovery-cpp_SOURCE_DIR}) endif(RS_BRODCAST_DISCOVERY) if(NOT RS_WARN_DEPRECATED) @@ -223,7 +391,7 @@ if(NOT RS_WARN_DEPRECATED) target_compile_options( ${PROJECT_NAME} PRIVATE -Wno-deprecated -Wno-deprecated-declarations ) -endif(RS_WARN_DEPRECATED) +endif(NOT RS_WARN_DEPRECATED) if(RS_WARN_LESS) add_compile_definitions(RS_NO_WARN_CPP) diff --git a/libretroshare/src/CMakeLists.txt b/libretroshare/src/CMakeLists.txt index 7bfbb52b1..d3ceffecf 100644 --- a/libretroshare/src/CMakeLists.txt +++ b/libretroshare/src/CMakeLists.txt @@ -5,6 +5,49 @@ # # SPDX-License-Identifier: CC0-1.0 +list( + APPEND RS_PUBLIC_HEADERS + retroshare/rsexpr.h + retroshare/rsgxsdistsync.h + retroshare/rsiface.h + retroshare/rsrtt.h + retroshare/rsbanlist.h + retroshare/rsconfig.h + retroshare/rsdisc.h + retroshare/rsflags.h + retroshare/rsgrouter.h + retroshare/rsgxsflags.h + retroshare/rsgxsservice.h + retroshare/rsgxstrans.h + retroshare/rsgxstunnel.h + retroshare/rsids.h + retroshare/rsnotify.h + retroshare/rsphoto.h + retroshare/rsplugin.h + retroshare/rsreputations.h + retroshare/rsservicecontrol.h + retroshare/rstokenservice.h + retroshare/rsturtle.h + retroshare/rsgossipdiscovery.h + retroshare/rsgxscommon.h + retroshare/rsposted.h + retroshare/rsstatus.h + retroshare/rsversion.h + retroshare/rsgxsifacehelper.h + retroshare/rshistory.h + retroshare/rsidentity.h + retroshare/rsmsgs.h + retroshare/rsgxschannels.h + retroshare/rsgxscircles.h + retroshare/rsgxsiface.h + retroshare/rsgxsifacetypes.h + retroshare/rstypes.h + retroshare/rsgxsforums.h + retroshare/rsevents.h + retroshare/rsfiles.h + retroshare/rsinit.h + retroshare/rspeers.h ) + list( APPEND RS_SOURCES chat/distantchat.cc @@ -17,6 +60,10 @@ list( crypto/rscrypto.cpp ) if(RS_BITDHT) + list( + APPEND RS_PUBLIC_HEADERS + retroshare/rsdht.h ) + list( APPEND RS_SOURCES dht/connectstatebox.cc @@ -80,6 +127,10 @@ list( gxstunnel/p3gxstunnel.cc ) if(RS_JSON_API) + list( + APPEND RS_PUBLIC_HEADERS + retroshare/rsjsonapi.h ) + list( APPEND RS_SOURCES jsonapi/jsonapi.cpp ) @@ -151,6 +202,7 @@ list( rsitems/rsrttitems.cc rsitems/rsserviceinfoitems.cc ) +#retroshare/rswiki.h #./rsitems/rswikiitems.cc #./rsitems/rswikiitems.h #./rsitems/rswireitems.h @@ -167,6 +219,7 @@ list( #./rsitems/rsposteditems.cc #./rsitems/rsposteditems.h #./rsitems/rswireitems.cc +#retroshare/rswire.h list( APPEND RS_SOURCES @@ -243,6 +296,10 @@ list( #./services/p3posted.h if(RS_BRODCAST_DISCOVERY) + list( + APPEND RS_PUBLIC_HEADERS + retroshare/rsbroadcastdiscovery.h ) + list( APPEND RS_SOURCES services/broadcastdiscoveryservice.cc ) @@ -255,9 +312,14 @@ list( tcponudp/tou.cc tcponudp/udppeer.cc tcponudp/bss_tou.cc - tcponudp/udprelay.cc - tcponudp/udpstunner.cc ) - + tcponudp/udprelay.cc ) + +if(RS_BITDHT_STUNNER) + list( + APPEND RS_SOURCES + tcponudp/udpstunner.cc ) +endif(RS_BITDHT_STUNNER) + list( APPEND RS_SOURCES turtle/rsturtleitem.cc diff --git a/libretroshare/src/jsonapi/README.adoc b/libretroshare/src/jsonapi/README.adoc new file mode 100644 index 000000000..64284b85c --- /dev/null +++ b/libretroshare/src/jsonapi/README.adoc @@ -0,0 +1,447 @@ +// SPDX-FileCopyrightText: (C) 2004-2019 Retroshare Team +// SPDX-License-Identifier: CC0-1.0 + +RetroShare JSON API +=================== + +:Cxx: C++ + +== How to use RetroShare JSON API + +Look for methods marked with +@jsonapi+ doxygen custom command into ++libretroshare/src/retroshare+. The method path is composed by service instance +pointer name like +rsGxsChannels+ for +RsGxsChannels+, and the method name like ++createGroup+ and pass the input paramethers as a JSON object. + +.Service instance pointer in rsgxschannels.h +[source,cpp] +-------------------------------------------------------------------------------- +/** + * Pointer to global instance of RsGxsChannels service implementation + * @jsonapi{development} + */ +extern RsGxsChannels* rsGxsChannels; +-------------------------------------------------------------------------------- + +.Method declaration in rsgxschannels.h +[source,cpp] +-------------------------------------------------------------------------------- + /** + * @brief Request channel creation. + * The action is performed asyncronously, so it could fail in a subsequent + * phase even after returning true. + * @jsonapi{development} + * @param[out] token Storage for RsTokenService token to track request + * status. + * @param[in] group Channel data (name, description...) + * @return false on error, true otherwise + */ + virtual bool createGroup(uint32_t& token, RsGxsChannelGroup& group) = 0; +-------------------------------------------------------------------------------- + +.paramethers.json +[source,json] +-------------------------------------------------------------------------------- +{ + "group":{ + "mMeta":{ + "mGroupName":"JSON test group", + "mGroupFlags":4, + "mSignFlags":520 + }, + "mDescription":"JSON test group description" + }, + "caller_data":"Here can go any kind of JSON data (even objects) that the caller want to get back together with the response" +} +-------------------------------------------------------------------------------- + +.Calling the JSON API with curl on the terminal +[source,bash] +-------------------------------------------------------------------------------- +curl -u $API_USER --data @paramethers.json http://127.0.0.1:9092/rsGxsChannels/createGroup +-------------------------------------------------------------------------------- + +.JSON API call result +[source,json] +-------------------------------------------------------------------------------- +{ + "caller_data": "Here can go any kind of JSON data (even objects) that the caller want to get back together with the response", + "retval": true, + "token": 3 +} +-------------------------------------------------------------------------------- + +Even if it is less efficient because of URL encoding HTTP +GET+ method is +supported too, so in cases where the client cannot use +POST+ she can still use ++GET+ taking care of encoding the JSON data. With +curl+ this can be done at +least in two different ways. + +.Calling the JSON API with GET method with curl on the terminal +[source,bash] +-------------------------------------------------------------------------------- +curl -u $API_USER --get --data-urlencode jsonData@paramethers.json \ + http://127.0.0.1:9092/rsGxsChannels/createGroup +-------------------------------------------------------------------------------- + +Letting +curl+ do the encoding is much more elegant but it is semantically +equivalent to the following. + +.Calling the JSON API with GET method and pre-encoded data with curl on the terminal +-------------------------------------------------------------------------------- +curl -u $API_USER http://127.0.0.1:9092/rsGxsChannels/createGroup?jsonData=%7B%0A%20%20%20%20%22group%22%3A%7B%0A%20%20%20%20%20%20%20%20%22mMeta%22%3A%7B%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mGroupName%22%3A%22JSON%20test%20group%22%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mGroupFlags%22%3A4%2C%0A%20%20%20%20%20%20%20%20%20%20%20%20%22mSignFlags%22%3A520%0A%20%20%20%20%20%20%20%20%7D%2C%0A%20%20%20%20%20%20%20%20%22mDescription%22%3A%22JSON%20test%20group%20description%22%0A%20%20%20%20%7D%2C%0A%20%20%20%20%22caller_data%22%3A%22Here%20can%20go%20any%20kind%20of%20JSON%20data%20%28even%20objects%29%20that%20the%20caller%20want%20to%20get%20back%20together%20with%20the%20response%22%0A%7D +-------------------------------------------------------------------------------- + +Note that using +GET+ method +?jsonData=+ and then the JSON data URL encoded are +added after the path in the HTTP request. + + +== JSON API authentication + +Most of JSON API methods require authentication as they give access to +RetroShare user data, and we don't want any application running on the system +eventually by other users be able to access private data indiscriminately. +JSON API support HTTP Basic as authentication scheme, this is enough as JSON API +server is intented for usage on the same system (127.0.0.1) not over an +untrusted network. +If you need to use JSON API over an untrusted network consider using a reverse +proxy with HTTPS such as NGINX in front of JSON API server. +If RetroShare login has been effectuated through the JSON API you can use your +location SSLID as username and your PGP password as credential for the JSON API, +but we suggests you use specific meaningful and human readable credentials for +each JSON API client so the human user can have better control over which client +can access the JSON API. + +.NewToken.json +[source,json] +-------------------------------------------------------------------------------- +{ + "token": "myNewUser:myNewPassword" +} +-------------------------------------------------------------------------------- + +.An authenticated client can authorize new tokens like this +-------------------------------------------------------------------------------- +curl -u $API_USER --data @NewToken.json http://127.0.0.1:9092/jsonApiServer/authorizeToken +-------------------------------------------------------------------------------- + +.An unauthenticated JSON API client can request access with +-------------------------------------------------------------------------------- +curl --data @NewToken.json http://127.0.0.1:9092/jsonApiServer/requestNewTokenAutorization +-------------------------------------------------------------------------------- + +When an unauthenticated client request his token to be authorized, JSON API +server will try to ask confirmation to the human user if possible through ++mNewAccessRequestCallback+, if it is not possible or the user didn't authorized +the token +false+ is returned. + + +== Offer new RetroShare services through JSON API + +To offer a retroshare service through the JSON API, first of all one need find +the global pointer to the service instance and document it in doxygen syntax, +plus marking with the custom doxygen command +@jsonapi{RS_VERSION}+ where ++RS_VERSION+ is the retroshare version in which this service became available +with the current semantic (major changes to the service semantic, changes the +meaning of the service itself, so the version should be updated in the +documentation in that case). + +.Service instance pointer in rsgxschannels.h +[source,cpp] +-------------------------------------------------------------------------------- +/** + * Pointer to global instance of RsGxsChannels service implementation + * @jsonapi{development} + */ +extern RsGxsChannels* rsGxsChannels; +-------------------------------------------------------------------------------- + + +Once the service instance itself is known to the JSON API you need to document +in doxygen syntax and mark with the custom doxygen command ++@jsonapi{RS_VERSION}+ the methods of the service that you want to make +available through JSON API. + +.Offering RsGxsChannels::getChannelDownloadDirectory in rsgxschannels.h +[source,cpp] +-------------------------------------------------------------------------------- + /** + * Get download directory for the given channel + * @jsonapi{development} + * @param[in] channelId id of the channel + * @param[out] directory reference to string where to store the path + * @return false on error, true otherwise + */ + virtual bool getChannelDownloadDirectory( const RsGxsGroupId& channelId, + std::string& directory ) = 0; +-------------------------------------------------------------------------------- + +For each paramether you must specify if it is used as input +@param[in]+ as +output +@param[out]+ or both +@param[inout]+. Paramethers and return value +types must be of a type supported by +RsTypeSerializer+ which already support +most basic types (+bool+, +std::string+...), +RsSerializable+ and containers of +them like +std::vector+. Paramethers passed by value and by +reference of those types are both supported, while passing by pointer is not +supported. If your paramether or return +class+/+struct+ type is not supported +yet by +RsTypeSerializer+ most convenient approach is to make it derive from ++RsSerializable+ and implement +serial_process+ method like I did with ++RsGxsChannelGroup+. + +.Deriving RsGxsChannelGroup from RsSerializable in rsgxschannels.h +[source,cpp] +-------------------------------------------------------------------------------- +struct RsGxsChannelGroup : RsSerializable +{ + RsGroupMetaData mMeta; + std::string mDescription; + RsGxsImage mImage; + + bool mAutoDownload; + + /// @see RsSerializable + virtual void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) + { + RS_SERIAL_PROCESS(mMeta); + RS_SERIAL_PROCESS(mDescription); + RS_SERIAL_PROCESS(mImage); + RS_SERIAL_PROCESS(mAutoDownload); + } +}; +-------------------------------------------------------------------------------- + +You can do the same recursively for any member of your +struct+ that is not yet +supported by +RsTypeSerializer+. + +Some Retroshare {Cxx} API functions are asyncronous, historically RetroShare +didn't follow a policy on how to expose asyncronous API so differents services +and some times even differents method of the same service follow differents +asyncronous patterns, thus making automatic generation of JSON API wrappers for +those methods impractical. Instead of dealing with all those differents patterns +I have chosed to support only one new pattern taking advantage of modern {Cxx}11 +and restbed features. On the {Cxx}11 side lambdas and +std::function+s are used, +on the restbed side Server Side Events are used to send asyncronous results. + +Lets see an example so it will be much esier to understand. + +.RsGxsChannels::turtleSearchRequest asyncronous API +[source,cpp] +-------------------------------------------------------------------------------- + /** + * @brief Request remote channels search + * @jsonapi{development} + * @param[in] matchString string to look for in the search + * @param multiCallback function that will be called each time a search + * result is received + * @param[in] maxWait maximum wait time in seconds for search results + * @return false on error, true otherwise + */ + virtual bool turtleSearchRequest( + const std::string& matchString, + const std::function& multiCallback, + std::time_t maxWait = 300 ) = 0; +-------------------------------------------------------------------------------- + ++RsGxsChannels::turtleSearchRequest(...)+ is an asyncronous method because it +send a channel search request on turtle network and then everytime a result is +received from the network +multiCallback+ is called and the result is passed as +parameter. To be supported by the automatic JSON API wrappers generator an +asyncronous method need a parameter of type +std::function+ called ++callback+ if the callback will be called only once or +multiCallback+ if the +callback is expected to be called more then once like in this case. +A second mandatory parameter is +maxWait+ of type +std::time_t+ it indicates the +maximum amount of time in seconds for which the caller is willing to wait for +results, in case the timeout is reached the callback will not be called anymore. + +[IMPORTANT] +================================================================================ ++callback+ and +multiCallback+ parameters documentation must *not* specify ++[in]+, +[out]+, +[inout]+, in Doxygen documentation as this would fool the +automatic wrapper generator, and ultimately break the compilation. +================================================================================ + +.RsFiles::turtleSearchRequest asyncronous JSON API usage example +[source,bash] +-------------------------------------------------------------------------------- +$ cat turtle_search.json +{ + "matchString":"linux" +} +$ curl --data @turtle_search.json http://127.0.0.1:9092/rsFiles/turtleSearchRequest +data: {"retval":true} + +data: {"results":[{"size":157631,"hash":"69709b4d01025584a8def5cd78ebbd1a3cf3fd05","name":"kill_bill_linux_1024x768.jpg"},{"size":192560,"hash":"000000000000000000009a93e5be8486c496f46c","name":"coffee_box_linux2.jpg"},{"size":455087,"hash":"9a93e5be8486c496f46c00000000000000000000","name":"Linux.png"},{"size":182004,"hash":"e8845280912ebf3779e400000000000000000000","name":"Linux_2_6.png"}]} + +data: {"results":[{"size":668,"hash":"e8845280912ebf3779e400000000000000000000","name":"linux.png"},{"size":70,"hash":"e8845280912ebf3779e400000000000000000000","name":"kali-linux-2016.2-amd64.txt.sha1sum"},{"size":3076767744,"hash":"e8845280912ebf3779e400000000000000000000","name":"kali-linux-2016.2-amd64.iso"},{"size":2780872,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.bin"},{"size":917504,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.lzma"},{"size":2278404096,"hash":"e8845280912ebf3779e400000000000000000000","name":"gentoo-linux-livedvd-amd64-multilib-20160704.iso"},{"size":151770333,"hash":"e8845280912ebf3779e400000000000000000000","name":"flashtool-0.9.23.0-linux.tar.7z"},{"size":2847372,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.elf"},{"size":1310720,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux.gz"},{"size":987809,"hash":"e8845280912ebf3779e400000000000000000000","name":"openwrt-ar71xx-generic-vmlinux-lzma.elf"}]} + +-------------------------------------------------------------------------------- + +By default JSON API methods requires client authentication and their wrappers +are automatically generated by +json-api-generator+. +In some cases methods need do be accessible without authentication such as ++rsLoginHelper/getLocations+ so in the doxygen documentaion they have the custom +command +@jsonapi{RS_VERSION,unauthenticated}+. +Other methods such as +/rsControl/rsGlobalShutDown+ need special care so they +are marked with the custom doxygen command +@jsonapi{RS_VERSION,manualwrapper}+ +and their wrappers are not automatically generated but written manually into ++JsonApiServer::JsonApiServer(...)+. + +== Quirks + +=== 64 bits integers handling + +While JSON doesn't have problems representing 64 bits integers JavaScript, Dart +and other languages are not capable to handle those numbers natively. +To overcome this limitation JSON API output 64 bit integers as an object with +two keys, one as proper integer and one as string representation. + +.JSON API 64 bit integer output example +[source,json] +-------------------------------------------------------------------------------- +"lobby_id": { "xint64": 6215642878098695544, "xstr64": "6215642878098695544" } +-------------------------------------------------------------------------------- + +So from languages that have proper 64bit integers support like Python or C++ one +better read from `xint64` which is represented as a JSON integer, from languages +where there is no proper 64bit integers support like JavaScript one can read from +`xstr64` which is represented as JSON string (note that the first is not wrapped +in "" while the latter is). + +When one input a 64bit integer into the JSON API it first try to parse it as if +it was sent the old way for retrocompatibility. + +.JSON API 64 bit integer deprecated format input example +[source,json] +-------------------------------------------------------------------------------- +"lobby_id":6215642878098695544 +-------------------------------------------------------------------------------- + +This way is *DEPRECATED* and may disappear in the future, it is TEMPORALLY kept +only for retrocompatibiliy with old clients. + +If retrocompatible parsing attempt fail then it try to parse with the new way +with proper JSON integer format. + +.JSON API 64 bit integer new proper integer format input example +[source,json] +-------------------------------------------------------------------------------- +lobby_id": { "xint64": 6215642878098695544 } +-------------------------------------------------------------------------------- + +If this fails then it try to parse with the new way with JSON string format. + +.JSON API 64 bit integer new string format input example +[source,json] +-------------------------------------------------------------------------------- +"lobby_id": { "xstr64": "6215642878098695544" } +-------------------------------------------------------------------------------- + +[WARNING] +================================================================================ +Clients written in languages without proper 64bit integers support must +use *ONLY* the string format otherwise they will send approximated values and +get unexpected results from the JSON API, because parsing will success but the +value will not be exactly the one you believe you sent. +================================================================================ + + +== A bit of history + +=== First writings about this + +The previous attempt of exposing a RetroShare JSON API is called +libresapi+ and +unfortunatley it requires a bunch of boilerplate code when we want to expose +something present in the {Cxx} API in the JSON API. + +As an example here you can see the libresapi that exposes part of the retroshare +chat {Cxx} API and lot of boilerplate code just to convert {Cxx} objects to JSON + +https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ChatHandler.cpp#L44 + +To avoid the {Cxx} to JSON and back conversion boilerplate code I have worked out +an extension to our {Cxx} serialization code so it is capable to serialize and +deserialize to JSON you can see it in this pull request + +https://github.com/RetroShare/RetroShare/pull/1155 + +So first step toward having a good API is to take advantage of the fact that RS +is now capable of converting C++ objects from and to JSON. + +The current API is accessible via HTTP and unix socket, there is no +authentication in both of them, so anyone having access to the HTTP server or to +the unix socket can access the API without extra restrictions. +Expecially for the HTTP API this is a big risk because also if the http server +listen on 127.0.0.1 every application on the machine (even rogue javascript +running on your web browser) can access that and for example on android it is +not safe at all (because of that I implemented the unix socket access so at +least in android API was reasonably safe) because of this. + +A second step to improve the API would be to implement some kind of API +authentication mechanism (it would be nice that the mechanism is handled at API +level and not at transport level so we can use it for any API trasport not just +HTTP for example) + +The HTTP server used by libresapi is libmicrohttpd server that is very minimal, +it doesn't provide HTTPS nor modern HTTP goodies, like server notifications, +websockets etc. because the lack of support we have a token polling mechanism in +libresapi to avoid polling for every thing but it is still ugly, so if we can +completely get rid of polling in the API that would be really nice. +I have done a crawl to look for a replacement and briefly looked at + +- https://www.gnu.org/software/libmicrohttpd/ +- http://wolkykim.github.io/libasyncd/ +- https://github.com/corvusoft/restbed +- https://code.facebook.com/posts/1503205539947302/introducing-proxygen-facebook-s-c-http-framework/ +- https://github.com/cmouse/yahttp + +taking in account a few metrics like modern HTTP goodies support, license, +platform support, external dependencies and documentation it seemed to me that +restbed is the more appropriate. + +Another source of boilerplate code into libresapi is the mapping between JSON +API requests and C++ API methods as an example you can look at this + +https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ChatHandler.cpp#L158 + +and this + +https://github.com/RetroShare/RetroShare/blob/v0.6.4/libresapi/src/api/ApiServer.cpp#L253 + +The abstract logic of this thing is, when libreasapi get a request like ++/chat/initiate_distant_chat+ then call ++ChatHandler::handleInitiateDistantChatConnexion+ which in turn is just a +wrapper of +RsMsgs::initiateDistantChatConnexion+ all this process is basically +implemented as boilerplate code and would be unnecessary in a smarter design of +the API because almost all the information needed is already present in the +C++ API +libretroshare/src/retroshare+. + +So a third step to improve the JSON API would be to remove this source of +boilerplate code by automatizing the mapping between C++ and JSON API call. + +This may result a little tricky as language parsing or other adevanced things +may be required. + +Hope this dive is useful for you + +Cheers + +G10h4ck + +=== Second writings about this + +I have been investigating a bit more about: +[verse, G10h4ck] +________________________________________________________________________________ +So a third step to improve the JSON API would be to remove this source of +boilerplate code by automatizing the mapping between C++ and JSON API call +________________________________________________________________________________ + +After spending some hours investigating this topic the most reasonable approach +seems to: + +1. Properly document headers in +libretroshare/src/retroshare/+ in doxygen syntax +specifying wihich params are input and/or output (doxygen sysntax for this is ++@param[in/out/inout]+) this will be the API documentation too. + +2. At compile time use doxygen to generate XML description of the headers and use +the XML to generate the JSON api server stub. +http://www.stack.nl/~dimitri/doxygen/manual/customize.html#xmlgenerator + +3. Enjoy diff --git a/libretroshare/src/jsonapi/async-method-wrapper-template.cpp.tmpl b/libretroshare/src/jsonapi/async-method-wrapper-template.cpp.tmpl new file mode 100644 index 000000000..29f53ad48 --- /dev/null +++ b/libretroshare/src/jsonapi/async-method-wrapper-template.cpp.tmpl @@ -0,0 +1,80 @@ +/******************************************************************************* + * RetroShare JSON API * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +registerHandler( "$%apiPath%$", + [this](const std::shared_ptr session) +{ + const std::multimap headers + { + { "Connection", "keep-alive" }, + { "Content-Type", "text/event-stream" } + }; + session->yield(rb::OK, headers); + + size_t reqSize = session->get_request()->get_header("Content-Length", 0); + session->fetch( reqSize, [this]( + const std::shared_ptr session, + const rb::Bytes& body ) + { + INITIALIZE_API_CALL_JSON_CONTEXT; + + if( !checkRsServicePtrReady( + $%instanceName%$, "$%instanceName%$", cAns, session ) ) + return; + +$%paramsDeclaration%$ + +$%inputParamsDeserialization%$ + + const std::weak_ptr weakService(mService); + const std::weak_ptr weakSession(session); + $%callbackName%$ = [weakService, weakSession]($%callbackParams%$) + { + auto session = weakSession.lock(); + if(!session || session->is_closed()) return; + + auto lService = weakService.lock(); + if(!lService || lService->is_down()) return; + +$%callbackParamsSerialization%$ + + std::stringstream sStream; + sStream << "data: " << compactJSON << ctx.mJson << "\n\n"; + const std::string message = sStream.str(); + + lService->schedule( [weakSession, message]() + { + auto session = weakSession.lock(); + if(!session || session->is_closed()) return; + session->yield(message); + $%sessionEarlyClose%$ + } ); + }; + +$%functionCall%$ + +$%outputParamsSerialization%$ + + // return them to the API caller + std::stringstream message; + message << "data: " << compactJSON << cAns.mJson << "\n\n"; + session->yield(message.str()); + $%sessionDelayedClose%$ + } ); +}, $%requiresAuth%$ ); diff --git a/libretroshare/src/jsonapi/jsonapi-generator-doxygen.conf b/libretroshare/src/jsonapi/jsonapi-generator-doxygen.conf new file mode 100644 index 000000000..0edd1de3e --- /dev/null +++ b/libretroshare/src/jsonapi/jsonapi-generator-doxygen.conf @@ -0,0 +1,230 @@ +DOXYFILE_ENCODING = UTF-8 +PROJECT_NAME = "libretroshare" + +ALIASES += jsonapi{1}="\xmlonly\endxmlonly" +ALIASES += jsonapi{2}="\xmlonly\endxmlonly" + +# If the INHERIT_DOCS tag is set to YES then an undocumented member inherits the +# documentation from any documented member that it re-implements. +# The default value is: YES. + +INHERIT_DOCS = YES + +# If the MARKDOWN_SUPPORT tag is enabled then doxygen pre-processes all comments +# according to the Markdown format, which allows for more readable +# documentation. See http://daringfireball.net/projects/markdown/ for details. +# The output of markdown processing is further processed by doxygen, so you can +# mix doxygen, HTML, and XML commands with Markdown formatting. Disable only in +# case of backward compatibilities issues. +# The default value is: YES. + +MARKDOWN_SUPPORT = YES + +# When enabled doxygen tries to link words that correspond to documented +# classes, or namespaces to their corresponding documentation. Such a link can +# be prevented in individual cases by putting a % sign in front of the word or +# globally by setting AUTOLINK_SUPPORT to NO. +# The default value is: YES. + +AUTOLINK_SUPPORT = YES + + +# If the EXTRACT_ALL tag is set to YES, doxygen will assume all entities in +# documentation are documented, even if no documentation was available. Private +# class members and static file members will be hidden unless the +# EXTRACT_PRIVATE respectively EXTRACT_STATIC tags are set to YES. +# Note: This will also disable the warnings about undocumented members that are +# normally produced when WARNINGS is set to YES. +# The default value is: NO. + +EXTRACT_ALL = YES + + +# If the HIDE_UNDOC_MEMBERS tag is set to YES, doxygen will hide all +# undocumented members inside documented classes or files. If set to NO these +# members will be included in the various overviews, but no documentation +# section is generated. This option has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_MEMBERS = NO + +# If the HIDE_UNDOC_CLASSES tag is set to YES, doxygen will hide all +# undocumented classes that are normally visible in the class hierarchy. If set +# to NO, these classes will be included in the various overviews. This option +# has no effect if EXTRACT_ALL is enabled. +# The default value is: NO. + +HIDE_UNDOC_CLASSES = NO + + +#--------------------------------------------------------------------------- +# Configuration options related to the input files +#--------------------------------------------------------------------------- + +# The INPUT tag is used to specify the files and/or directories that contain +# documented source files. You may enter file names like myfile.cpp or +# directories like /usr/src/myproject. Separate the files or directories with +# spaces. See also FILE_PATTERNS and EXTENSION_MAPPING +# Note: If this tag is empty the current directory is searched. + +#INPUT = + +# This tag can be used to specify the character encoding of the source files +# that doxygen parses. Internally doxygen uses the UTF-8 encoding. Doxygen uses +# libiconv (or the iconv built into libc) for the transcoding. See the libiconv +# documentation (see: https://www.gnu.org/software/libiconv/) for the list of +# possible encodings. +# The default value is: UTF-8. + +INPUT_ENCODING = UTF-8 + +# If the value of the INPUT tag contains directories, you can use the +# FILE_PATTERNS tag to specify one or more wildcard patterns (like *.cpp and +# *.h) to filter out the source-files in the directories. +# +# Note that for custom extensions or not directly supported extensions you also +# need to set EXTENSION_MAPPING for the extension otherwise the files are not +# read by doxygen. +# +# If left blank the following patterns are tested:*.c, *.cc, *.cxx, *.cpp, +# *.c++, *.java, *.ii, *.ixx, *.ipp, *.i++, *.inl, *.idl, *.ddl, *.odl, *.h, +# *.hh, *.hxx, *.hpp, *.h++, *.cs, *.d, *.php, *.php4, *.php5, *.phtml, *.inc, +# *.m, *.markdown, *.md, *.mm, *.dox, *.py, *.pyw, *.f90, *.f95, *.f03, *.f08, +# *.f, *.for, *.tcl, *.vhd, *.vhdl, *.ucf and *.qsf. + +FILE_PATTERNS = *.c \ + *.cc \ + *.cxx \ + *.cpp \ + *.c++ \ + *.java \ + *.ii \ + *.ixx \ + *.ipp \ + *.i++ \ + *.inl \ + *.idl \ + *.ddl \ + *.odl \ + *.h \ + *.hh \ + *.hxx \ + *.hpp \ + *.h++ \ + *.cs \ + *.d \ + *.php \ + *.php4 \ + *.php5 \ + *.phtml \ + *.inc \ + *.m \ + *.markdown \ + *.md \ + *.mm \ + *.dox \ + *.py \ + *.pyw \ + *.f90 \ + *.f95 \ + *.f03 \ + *.f08 \ + *.f \ + *.for \ + *.tcl \ + *.vhd \ + *.vhdl \ + *.ucf \ + *.qsf + +# The RECURSIVE tag can be used to specify whether or not subdirectories should +# be searched for input files as well. +# The default value is: NO. + +RECURSIVE = YES + +# The EXCLUDE tag can be used to specify files and/or directories that should be +# excluded from the INPUT source files. This way you can easily exclude a +# subdirectory from a directory tree whose root is specified with the INPUT tag. +# +# Note that relative paths are relative to the directory from which doxygen is +# run. + +EXCLUDE = + +# The EXCLUDE_SYMLINKS tag can be used to select whether or not files or +# directories that are symbolic links (a Unix file system feature) are excluded +# from the input. +# The default value is: NO. + +EXCLUDE_SYMLINKS = NO + +# If the value of the INPUT tag contains directories, you can use the +# EXCLUDE_PATTERNS tag to specify one or more wildcard patterns to exclude +# certain files from those directories. +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories for example use the pattern */test/* + +EXCLUDE_PATTERNS = + +# The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names +# (namespaces, classes, functions, etc.) that should be excluded from the +# output. The symbol name can be a fully qualified name, a word, or if the +# wildcard * is used, a substring. Examples: ANamespace, AClass, +# AClass::ANamespace, ANamespace::*Test +# +# Note that the wildcards are matched against the file with absolute path, so to +# exclude all test directories use the pattern */test/* + +EXCLUDE_SYMBOLS = + + +#--------------------------------------------------------------------------- +# Configuration options related to the HTML output +#--------------------------------------------------------------------------- + +# If the GENERATE_HTML tag is set to YES, doxygen will generate HTML output +# The default value is: YES. + +GENERATE_HTML = NO + +# If the GENERATE_LATEX tag is set to YES, doxygen will generate LaTeX output. +# The default value is: YES. + +GENERATE_LATEX = NO + +# If the GENERATE_XML tag is set to YES, doxygen will generate an XML file that +# captures the structure of the code including all documentation. +# The default value is: NO. + +GENERATE_XML = YES + + +#--------------------------------------------------------------------------- +# Configuration options related to the preprocessor +#--------------------------------------------------------------------------- + +# If the ENABLE_PREPROCESSING tag is set to YES, doxygen will evaluate all +# C-preprocessor directives found in the sources and include files. +# The default value is: YES. + +ENABLE_PREPROCESSING = YES + +# If the MACRO_EXPANSION tag is set to YES, doxygen will expand all macro names +# in the source code. If set to NO, only conditional compilation will be +# performed. Macro expansion can be done in a controlled way by setting +# EXPAND_ONLY_PREDEF to YES. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +MACRO_EXPANSION = NO + +# If the EXPAND_ONLY_PREDEF and MACRO_EXPANSION tags are both set to YES then +# the macro expansion is limited to the macros specified with the PREDEFINED and +# EXPAND_AS_DEFINED tags. +# The default value is: NO. +# This tag requires that the tag ENABLE_PREPROCESSING is set to YES. + +EXPAND_ONLY_PREDEF = NO + diff --git a/libretroshare/src/jsonapi/jsonapi-generator.py b/libretroshare/src/jsonapi/jsonapi-generator.py new file mode 100755 index 000000000..accf9a0cd --- /dev/null +++ b/libretroshare/src/jsonapi/jsonapi-generator.py @@ -0,0 +1,365 @@ +#!/usr/bin/python3 + +# RetroShare JSON API generator +# +# Copyright (C) 2019 selankon +# Copyright (C) 2021 Gioacchino Mazzurco +# Copyright (C) 2021 Asociación Civil Altermundi +# +# This program is free software: you can redistribute it and/or modify it under +# the terms of the GNU Affero General Public License as published by the +# Free Software Foundation, version 3. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. +# See the GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License along +# with this program. If not, see +# +# SPDX-FileCopyrightText: Retroshare Team +# SPDX-License-Identifier: AGPL-3.0-only + + +# Original idea and implementation by G10h4ck (jsonapi-generator.cpp) +# Initial python reimplementation by Sehraf +# +# This python 3 script has superseded the original C++/Qt implementation this +# and is now used at build time in without depending on Qt. + +import os +import sys + +import xml.etree.ElementTree as ET +from string import Template + + +class MethodParam: + _type = '' + _name = '' + _defval = '' + _in = False + _out = False + _isMultiCallback = False + _isSingleCallback = False + + +class TemplateOwn(Template): + delimiter = '$%' + pattern = ''' + \$%(?: + (?P\$\%) | # Escape sequence of two delimiters + (?P[_a-z][_a-z0-9]*)%\$ | # delimiter and a Python identifier + {(?P[_a-z][_a-z0-9]*)} | # delimiter and a braced identifier + (?P) # Other ill-formed delimiter exprs + ) + ''' + + +def getText(e): + return "".join(e.itertext()) + + +def processFile(file): + try: + dom1 = ET.parse(file).getroot() + except FileNotFoundError: + print('Can\'t open:', file) + + headerFileInfo = dom1[0].findall('location')[0].attrib['file'] + headerRelPath = os.path.dirname(headerFileInfo).split('/')[-1] + '/' + os.path.basename(headerFileInfo) + + for sectDef in dom1.findall('.//memberdef'): + if sectDef.attrib['kind'] != 'variable' or sectDef.find('.//jsonapi') == None: + continue + + instanceName = sectDef.find('name').text + typeName = sectDef.find('type/ref').text + + typeFilePath = sectDef.find('type/ref').attrib['refid'] + + try: + dom2 = ET.parse(doxPrefix + typeFilePath + '.xml').getroot() + except FileNotFoundError: + print('Can\'t open:', doxPrefix + typeFilePath + '.xml') + + for member in dom2.findall('.//member'): + refid = member.attrib['refid'] + methodName = member.find('name').text + + requiresAuth = True + + defFilePath = refid.split('_')[0] + '.xml' + defFile = defFilePath + + print('Looking for', typeName, methodName, 'into', typeFilePath) + + try: + defDoc = ET.parse(doxPrefix + defFilePath).getroot() + except FileNotFoundError: + print('Can\'t open:', doxPrefix + defFilePath) + + memberdef = None + for tmpMBD in defDoc.findall('.//memberdef'): + tmpId = tmpMBD.attrib['id'] + tmpKind = tmpMBD.attrib['kind'] + tmpJsonApiTagList = tmpMBD.findall('.//jsonapi') + + if len(tmpJsonApiTagList) != 0 and tmpId == refid and tmpKind == 'function': + tmpJsonApiTag = tmpJsonApiTagList[0] + + tmpAccessValue = None + if 'access' in tmpJsonApiTag.attrib: + tmpAccessValue = tmpJsonApiTag.attrib['access'] + + requiresAuth = 'unauthenticated' != tmpAccessValue; + + if 'manualwrapper' != tmpAccessValue: + memberdef = tmpMBD + + break + + if memberdef == None: + continue + + apiPath = '/' + instanceName + '/' + methodName + + retvalType = getText(memberdef.find('type')) + # Apparently some xml declarations include new lines ('\n') and/or multiple spaces + # Strip them using python magic + retvalType = ' '.join(retvalType.split()) + + paramsMap = {} + orderedParamNames = [] + + hasInput = False + hasOutput = False + hasSingleCallback = False + hasMultiCallback = False + callbackName = '' + callbackParams = '' + + for tmpPE in memberdef.findall('param'): + mp = MethodParam() + + pName = getText(tmpPE.find('declname')) + tmpDefval = tmpPE.find('defval') + mp._defval = getText(tmpDefval) if tmpDefval != None else '' + pType = getText(tmpPE.find('type')) + + if pType.startswith('const '): pType = pType[6:] + if pType.startswith('std::function'): + if pType.endswith('&'): pType = pType[:-1] + if pName.startswith('multiCallback'): + mp._isMultiCallback = True + hasMultiCallback = True + elif pName.startswith('callback'): + mp._isSingleCallback = True + hasSingleCallback = True + callbackName = pName + callbackParams = pType + else: + pType = pType.replace('&', '').replace(' ', '') + + # Apparently some xml declarations include new lines ('\n') and/or multiple spaces + # Strip them using python magic + pType = ' '.join(pType.split()) + mp._defval = ' '.join(mp._defval.split()) + + mp._type = pType + mp._name = pName + + paramsMap[pName] = mp + orderedParamNames.append(pName) + + for tmpPN in memberdef.findall('.//parametername'): + tmpParam = paramsMap[tmpPN.text] + tmpD = tmpPN.attrib['direction'] if 'direction' in tmpPN.attrib else '' + + if 'in' in tmpD: + tmpParam._in = True + hasInput = True + if 'out' in tmpD: + tmpParam._out = True + hasOutput = True + + # Params sanity check + for pmKey in paramsMap: + pm = paramsMap[pmKey] + if not (pm._isMultiCallback or pm._isSingleCallback or pm._in or pm._out): + print('ERROR', 'Parameter:', pm._name, 'of:', apiPath, + 'declared in:', headerRelPath, + 'miss doxygen parameter direction attribute!', + defFile) + sys.exit() + + functionCall = '\t\t' + if retvalType != 'void': + functionCall += retvalType + ' retval = ' + hasOutput = True + functionCall += instanceName + '->' + methodName + '(' + functionCall += ', '.join(orderedParamNames) + ');\n' + + print(instanceName, apiPath, retvalType, typeName, methodName) + for pn in orderedParamNames: + mp = paramsMap[pn] + print('\t', mp._type, mp._name, mp._in, mp._out) + + inputParamsDeserialization = '' + if hasInput: + inputParamsDeserialization += '\t\t{\n' + inputParamsDeserialization += '\t\t\tRsGenericSerializer::SerializeContext& ctx(cReq);\n' + inputParamsDeserialization += '\t\t\tRsGenericSerializer::SerializeJob j(RsGenericSerializer::FROM_JSON);\n'; + + outputParamsSerialization = '' + if hasOutput: + outputParamsSerialization += '\t\t{\n' + outputParamsSerialization += '\t\t\tRsGenericSerializer::SerializeContext& ctx(cAns);\n' + outputParamsSerialization += '\t\t\tRsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON);\n'; + + paramsDeclaration = '' + for pn in orderedParamNames: + mp = paramsMap[pn] + paramsDeclaration += '\t\t' + mp._type + ' ' + mp._name + if mp._defval != '': + paramsDeclaration += ' = ' + mp._defval + paramsDeclaration += ';\n' + if mp._in: + inputParamsDeserialization += '\t\t\tRS_SERIAL_PROCESS(' + inputParamsDeserialization += mp._name + ');\n' + if mp._out: + outputParamsSerialization += '\t\t\tRS_SERIAL_PROCESS(' + outputParamsSerialization += mp._name + ');\n' + + if hasInput: + inputParamsDeserialization += '\t\t}\n' + if retvalType != 'void': + outputParamsSerialization += '\t\t\tRS_SERIAL_PROCESS(retval);\n' + if hasOutput: + outputParamsSerialization += '\t\t}\n' + + captureVars = '' + + sessionEarlyClose = '' + if hasSingleCallback: + sessionEarlyClose = 'session->close();' + + sessionDelayedClose = '' + if hasMultiCallback: + sessionDelayedClose = """ + RsThread::async( [=]() + { + std::this_thread::sleep_for( + std::chrono::seconds(maxWait+120) ); + auto lService = weakService.lock(); + if(!lService || lService->is_down()) return; + lService->schedule( [=]() + { + auto session = weakSession.lock(); + if(session && session->is_open()) + session->close(); + } ); + } ); +""" + captureVars = 'this' + + callbackParamsSerialization = '' + + if hasSingleCallback or hasMultiCallback or (callbackParams.find('(') + 2 < callbackParams.find(')')): + cbs = '' + + callbackParams = callbackParams.split('(')[1] + callbackParams = callbackParams.split(')')[0] + + cbs += '\t\t\tRsGenericSerializer::SerializeContext ctx;\n' + + for cbPar in callbackParams.split(','): + isConst = cbPar.startswith('const ') + pSep = ' ' + isRef = '&' in cbPar + if isRef: pSep = '&' + sepIndex = cbPar.rfind(pSep) + 1 + cpt = cbPar[0:sepIndex][6:] + cpn = cbPar[sepIndex:] + + cbs += '\t\t\tRsTypeSerializer::serial_process(' + cbs += 'RsGenericSerializer::TO_JSON, ctx, ' + if isConst: + cbs += 'const_cast<' + cbs += cpt + cbs += '>(' + cbs += cpn + if isConst: cbs += ')' + cbs += ', "' + cbs += cpn + cbs += '" );\n' + + callbackParamsSerialization += cbs + + substitutionsMap = dict() + substitutionsMap['paramsDeclaration'] = paramsDeclaration + substitutionsMap['inputParamsDeserialization'] = inputParamsDeserialization + substitutionsMap['outputParamsSerialization'] = outputParamsSerialization + substitutionsMap['instanceName'] = instanceName + substitutionsMap['functionCall'] = functionCall + substitutionsMap['apiPath'] = apiPath + substitutionsMap['sessionEarlyClose'] = sessionEarlyClose + substitutionsMap['sessionDelayedClose'] = sessionDelayedClose + substitutionsMap['captureVars'] = captureVars + substitutionsMap['callbackName'] = callbackName + substitutionsMap['callbackParams'] = callbackParams + substitutionsMap['callbackParamsSerialization'] = callbackParamsSerialization + substitutionsMap['requiresAuth'] = 'true' if requiresAuth else 'false' + + # print(substitutionsMap) + + templFilePath = sourcePath + if hasMultiCallback or hasSingleCallback: + templFilePath += '/async-method-wrapper-template.cpp.tmpl' + else: + templFilePath += '/method-wrapper-template.cpp.tmpl' + + templFile = open(templFilePath, 'r') + wrapperDef = TemplateOwn(templFile.read()) + + tmp = wrapperDef.substitute(substitutionsMap) + wrappersDefFile.write(tmp) + + cppApiIncludesSet.add('#include "' + headerRelPath + '"\n') + + +if len(sys.argv) != 3: + print('Usage:', sys.argv[0], 'SOURCE_PATH OUTPUT_PATH Got:', sys.argv[:]) + sys.exit(-1) + +sourcePath = str(sys.argv[1]) +outputPath = str(sys.argv[2]) +doxPrefix = outputPath + '/xml/' + +try: + wrappersDefFile = open(outputPath + '/jsonapi-wrappers.inl', 'w') +except FileNotFoundError: + print('Can\'t open:', outputPath + '/jsonapi-wrappers.inl') + +try: + cppApiIncludesFile = open(outputPath + '/jsonapi-includes.inl', 'w'); +except FileNotFoundError: + print('Can\'t open:', outputPath + '/jsonapi-includes.inl') + +cppApiIncludesSet = set() + +filesIterator = None +try: + filesIterator = os.listdir(doxPrefix) +except FileNotFoundError: + print("Doxygen xml output dir not found: ", doxPrefix) + os.exit(-1) + +for file in filesIterator: + if file.endswith("8h.xml"): + processFile(os.path.join(doxPrefix, file)) + + +for incl in cppApiIncludesSet: + cppApiIncludesFile.write(incl) diff --git a/libretroshare/src/jsonapi/method-wrapper-template.cpp.tmpl b/libretroshare/src/jsonapi/method-wrapper-template.cpp.tmpl new file mode 100644 index 000000000..a3927fba9 --- /dev/null +++ b/libretroshare/src/jsonapi/method-wrapper-template.cpp.tmpl @@ -0,0 +1,50 @@ +/******************************************************************************* + * RetroShare JSON API * + * * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +registerHandler( "$%apiPath%$", + [](const std::shared_ptr session) +{ + size_t reqSize = session->get_request()->get_header("Content-Length", 0); + session->fetch( reqSize, []( + const std::shared_ptr session, + const rb::Bytes& body ) + { + INITIALIZE_API_CALL_JSON_CONTEXT; + + if( !checkRsServicePtrReady( + $%instanceName%$, "$%instanceName%$", cAns, session ) ) + return; + +$%paramsDeclaration%$ + + // deserialize input parameters from JSON +$%inputParamsDeserialization%$ + + // call retroshare C++ API +$%functionCall%$ + + // serialize out parameters and return value to JSON +$%outputParamsSerialization%$ + + // return them to the API caller + DEFAULT_API_CALL_JSON_RETURN(rb::OK); + } ); +}, $%requiresAuth%$ ); +