Improve API

Manually expose /rsFiles/getFileData to stream/preview files
Automatically expose a bunch of methods via JSON API
Implement serial_process for std::pair

libretroshare/src/ft/ftserver.cc

@@ -425,7 +425,7 @@ void ftServer::requestDirUpdate(void *ref)
 }
 
 /* Directory Handling */
-bool ftServer::setDownloadDirectory(std::string path)
+bool ftServer::setDownloadDirectory(const std::string& path)
 {
 	return mFtController->setDownloadDirectory(path);
 }
@@ -435,7 +435,7 @@ std::string ftServer::getDownloadDirectory()
 	return mFtController->getDownloadDirectory();
 }
 
-bool ftServer::setPartialsDirectory(std::string path)
+bool ftServer::setPartialsDirectory(const std::string& path)
 {
 	return mFtController->setPartialsDirectory(path);
 }

libretroshare/src/ft/ftserver.h

@@ -205,8 +205,8 @@ public:
          * Directory Handling
          ***/
     virtual void requestDirUpdate(void *ref) ;			// triggers the update of the given reference. Used when browsing.
-    virtual bool setDownloadDirectory(std::string path);
-    virtual bool setPartialsDirectory(std::string path);
+	virtual bool setDownloadDirectory(const std::string& path);
+	virtual bool setPartialsDirectory(const std::string& path);
     virtual std::string getDownloadDirectory();
     virtual std::string getPartialsDirectory();
 

libretroshare/src/jsonapi/jsonapi.cpp

@@ -21,9 +21,11 @@
 #include <sstream>
 #include <memory>
 #include <restbed>
+#include <vector>
 
 #include "util/rsjson.h"
-#include "retroshare/rsgxschannels.h"
+#include "retroshare/rsfiles.h"
+#include "util/radix64.h"
 
 // Generated at compile time
 #include "jsonapi-includes.inl"
@@ -38,6 +40,80 @@ JsonApiServer::JsonApiServer(
 		shutdown();
 	});
 
+	registerHandler("/rsFiles/getFileData",
+	                [](const std::shared_ptr<rb::Session> session)
+	{
+		size_t reqSize = session->get_request()->get_header("Content-Length", 0);
+		session->fetch( reqSize, [](
+		                const std::shared_ptr<rb::Session> session,
+		                const rb::Bytes& body )
+		{
+			RsGenericSerializer::SerializeContext cReq(
+			            nullptr, 0,
+			            RsGenericSerializer::SERIALIZATION_FLAG_YIELDING );
+			RsJson& jReq(cReq.mJson);
+			jReq.Parse(reinterpret_cast<const char*>(body.data()), body.size());
+
+			RsGenericSerializer::SerializeContext cAns;
+			RsJson& jAns(cAns.mJson);
+
+			// if caller specified caller_data put it back in the answhere
+			const char kcd[] = "caller_data";
+			if(jReq.HasMember(kcd))
+				jAns.AddMember(kcd, jReq[kcd], jAns.GetAllocator());
+
+			RsFileHash hash;
+			uint64_t offset;
+			uint32_t requested_size;
+			bool retval = false;
+			std::string errorMessage;
+			std::string base64data;
+
+			// deserialize input parameters from JSON
+			{
+				RsGenericSerializer::SerializeContext& ctx(cReq);
+				RsGenericSerializer::SerializeJob j(RsGenericSerializer::FROM_JSON);
+				RS_SERIAL_PROCESS(hash);
+				RS_SERIAL_PROCESS(offset);
+				RS_SERIAL_PROCESS(requested_size);
+			}
+
+			if(requested_size > 10485760)
+				errorMessage = "requested_size is too big! Better less then 1M";
+			else
+			{
+				std::vector<uint8_t> buffer(requested_size);
+
+				// call retroshare C++ API
+				retval = rsFiles->getFileData(
+				            hash, offset, requested_size, buffer.data());
+
+				Radix64::encode(buffer.data(), requested_size, base64data);
+			}
+
+			// serialize out parameters and return value to JSON
+			{
+				RsGenericSerializer::SerializeContext& ctx(cAns);
+				RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON);
+				RS_SERIAL_PROCESS(retval);
+				RS_SERIAL_PROCESS(requested_size);
+				RS_SERIAL_PROCESS(base64data);
+				if(!errorMessage.empty()) RS_SERIAL_PROCESS(errorMessage);
+			}
+
+			// return them to the API caller
+			std::stringstream ss;
+			ss << jAns;
+			std::string&& ans(ss.str());
+			const std::multimap<std::string, std::string> headers
+			{
+				{ "Content-Type", "text/json" },
+				{ "Content-Length", std::to_string(ans.length()) }
+			};
+			session->close(rb::OK, ans, headers);
+		} );
+	});
+
 // Generated at compile time
 #include "jsonapi-wrappers.inl"
 }

libretroshare/src/retroshare/rsfiles.h

@@ -19,8 +19,7 @@
  * along with this program. If not, see <https://www.gnu.org/licenses/>.       *
  *                                                                             *
  *******************************************************************************/
-#ifndef RS_FILES_GUI_INTERFACE_H
-#define RS_FILES_GUI_INTERFACE_H
+#pragma once
 
 #include <list>
 #include <iostream>
@@ -186,31 +185,61 @@ public:
 	RsFiles() {}
 	virtual ~RsFiles() {}
 
-        /**
-         * Provides file data for the gui: media streaming or rpc clients.
-         * It may return unverified chunks. This allows streaming without having to wait for hashes or completion of the file.
-         * This function returns an unspecified amount of bytes. Either as much data as available or a sensible maximum. Expect a block size of around 1MiB.
-         * To get more data, call this function repeatedly with different offsets.
-         * Returns false in case
-         * - the files is not available on the local node
-         * - not downloading
-         * - the requested data was not downloaded yet
-         * - end of file was reached
-         * @param hash hash of a file. The file has to be available on this node or it has to be in downloading state.
-         * @param offset where the desired block starts
-     * @param requested_size size of pre-allocated data. Will be updated by the function.
-         * @param data pre-allocated memory chunk of size 'requested_size' by the client
-         */
-        virtual bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data)=0;
+	/**
+	 * Provides file data for the gui, media streaming or rpc clients.
+	 * It may return unverified chunks. This allows streaming without having to
+	 * wait for hashes or completion of the file.
+	 * This function returns an unspecified amount of bytes. Either as much data
+	 * as available or a sensible maximum. Expect a block size of around 1MiB.
+	 * To get more data, call this function repeatedly with different offsets.
+	 * jsonapi{development} note the missing @ the wrapper for this is written
+	 * manually not autogenerated @see JsonApiServer.
+	 * @param[in] hash hash of the file. The file has to be available on this node
+	 * 	or it has to be in downloading state.
+	 * @param[in] offset where the desired block starts
+	 * @param[inout] requested_size size of pre-allocated data. Will be updated
+	 * 	by the function.
+	 * @param data pre-allocated memory chunk of size 'requested_size' by the
+	 * 	client
+	 * @return Returns false in case
+	 * 	- the files is not available on the local node
+	 * 	- not downloading
+	 * 	- the requested data was not downloaded yet
+	 * 	- end of file was reached
+	 */
+	virtual bool getFileData( const RsFileHash& hash, uint64_t offset,
+	                          uint32_t& requested_size, uint8_t* data ) = 0;
 
 		/***
 		 *  Control of Downloads.
 		 ***/
 
 		virtual bool alreadyHaveFile(const RsFileHash& hash, FileInfo &info) = 0;
-		/// Returns false is we already have the file. Otherwise, initiates the dl and returns true.
-		virtual bool FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list<RsPeerId>& srcIds) = 0;
-		virtual bool FileCancel(const RsFileHash& hash) = 0;
+
+	/**
+	 * @brief Initiate downloading of a file
+	 * @jsonapi{development}
+	 * @param[in] fileName
+	 * @param[in] hash
+	 * @param[in] size
+	 * @param[in] destPath in not empty specify a destination path
+	 * @param[in] flags you usually want RS_FILE_REQ_ANONYMOUS_ROUTING
+	 * @param[in] srcIds eventually specify known sources
+	 * @return false if we already have the file, true otherwhise
+	 */
+	virtual bool FileRequest(
+	        const std::string& fileName, const RsFileHash& hash, uint64_t size,
+	        const std::string& destPath, TransferRequestFlags flags,
+	        const std::list<RsPeerId>& srcIds ) = 0;
+
+	/**
+	 * @brief Cancel file downloading
+	 * @jsonapi{development}
+	 * @param[in] hash
+	 * @return false if the file is not in the download queue, true otherwhise
+	 */
+	virtual bool FileCancel(const RsFileHash& hash) = 0;
+
 		virtual bool setDestinationDirectory(const RsFileHash& hash,const std::string& new_path) = 0;
 		virtual bool setDestinationName(const RsFileHash& hash,const std::string& new_name) = 0;
 		virtual bool setChunkStrategy(const RsFileHash& hash,FileChunksInfo::ChunkStrategy) = 0;
@@ -293,10 +322,35 @@ public:
 		 ***/
         virtual void requestDirUpdate(void *ref) =0 ;			// triggers the update of the given reference. Used when browsing.
 
-		virtual bool setDownloadDirectory(std::string path) = 0;
-		virtual bool setPartialsDirectory(std::string path) = 0;
-		virtual std::string getDownloadDirectory() = 0;
-		virtual std::string getPartialsDirectory() = 0;
+	/**
+	 * @brief Set default complete downloads directory
+	 * @jsonapi{development}
+	 * @param[in] path directory path
+	 * @return false if something failed, true otherwhise
+	 */
+	virtual bool setDownloadDirectory(const std::string& path) = 0;
+
+	/**
+	 * @brief Set partial downloads directory
+	 * @jsonapi{development}
+	 * @param[in] path directory path
+	 * @return false if something failed, true otherwhise
+	 */
+	virtual bool setPartialsDirectory(const std::string& path) = 0;
+
+	/**
+	 * @brief Get default complete downloads directory
+	 * @jsonapi{development}
+	 * @return default completed downloads directory path
+	 */
+	virtual std::string getDownloadDirectory() = 0;
+
+	/**
+	 * @brief Get partial downloads directory
+	 * @jsonapi{development}
+	 * @return partials downloads directory path
+	 */
+	virtual std::string getPartialsDirectory() = 0;
 
 	/**
 	 * @brief Get list of current shared directories
@@ -315,7 +369,7 @@ public:
 	virtual bool setSharedDirectories(const std::list<SharedDirInfo>& dirs) = 0;
 
 	/**
-	 * @brief Add shared directoryaddSharedDirectory
+	 * @brief Add shared directory
 	 * @jsonapi{development}
 	 * @param[in] dir directory to share with sharing options
 	 * @return false if something failed, true otherwhise
@@ -326,7 +380,7 @@ public:
 	 * @brief Updates shared directory sharing flags.
 	 * The directory should be already shared!
 	 * @jsonapi{development}
-	 * @param dir[in] Shared directory with updated sharing options
+	 * @param[in] dir Shared directory with updated sharing options
 	 * @return false if something failed, true otherwhise
 	 */
 	virtual bool updateShareFlags(const SharedDirInfo& dir) = 0;
@@ -334,7 +388,7 @@ public:
 	/**
 	 * @brief Remove directory from shared list
 	 * @jsonapi{development}
-	 * @param dir[in] Path of the directory to remove from shared list
+	 * @param[in] dir Path of the directory to remove from shared list
 	 * @return false if something failed, true otherwhise
 	 */
 	virtual bool removeSharedDirectory(std::string dir) = 0;
@@ -359,8 +413,4 @@ public:
 
 		virtual bool	ignoreDuplicates() = 0;
 		virtual void 	setIgnoreDuplicates(bool ignore) = 0;
-
 };
-
-
-#endif

libretroshare/src/retroshare/rsgxschannels.h

@@ -146,6 +146,13 @@ public:
 	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts) = 0;
 	virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
 
+	/**
+	 * @brief toggle message read status
+	 * @jsonapi{development}
+	 * @param[out] token GXS token queue token
+	 * @param[in] msgId
+	 * @param[in] read
+	 */
 	virtual void setMessageReadStatus(
 	        uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0;
 

libretroshare/src/serialiser/rstypeserializer.h

@@ -363,6 +363,86 @@ struct RsTypeSerializer
 		}
 	}
 
+	/// std::pair<T,U>
+	template<typename T, typename U>
+	static void serial_process( RsGenericSerializer::SerializeJob j,
+	                            RsGenericSerializer::SerializeContext& ctx,
+	                            std::pair<T,U>& p,
+	                            const std::string& memberName )
+	{
+		switch(j)
+		{
+		case RsGenericSerializer::SIZE_ESTIMATE: // fallthrough
+		case RsGenericSerializer::DESERIALIZE: // fallthrough
+		case RsGenericSerializer::SERIALIZE: // fallthrough
+		case RsGenericSerializer::PRINT:
+			serial_process(j, ctx, p.first, "pair::first");
+			serial_process(j, ctx, p.second, "pair::second");
+			break;
+		case RsGenericSerializer::TO_JSON:
+		{
+			RsJson& jDoc(ctx.mJson);
+			RsJson::AllocatorType& allocator = jDoc.GetAllocator();
+
+			// Reuse allocator to avoid deep copy later
+			RsGenericSerializer::SerializeContext lCtx(
+			            nullptr, 0, ctx.mFlags, &allocator );
+
+			serial_process(j, ctx, p.first, "first");
+			serial_process(j, ctx, p.second, "second");
+
+			rapidjson::Value key;
+			key.SetString(memberName.c_str(), memberName.length(), allocator);
+
+			/* Because the passed allocator is reused it doesn't go out of scope
+			 * and there is no need of deep copy and we can take advantage of
+			 * the much faster rapidjson move semantic */
+			jDoc.AddMember(key, lCtx.mJson, allocator);
+
+			ctx.mOk = ctx.mOk && lCtx.mOk;
+			break;
+		}
+		case RsGenericSerializer::FROM_JSON:
+		{
+			RsJson& jDoc(ctx.mJson);
+			const char* mName = memberName.c_str();
+			bool hasMember = jDoc.HasMember(mName);
+			bool yielding = ctx.mFlags &
+			        RsGenericSerializer::SERIALIZATION_FLAG_YIELDING;
+
+			if(!hasMember)
+			{
+				if(!yielding)
+				{
+					std::cerr << __PRETTY_FUNCTION__ << " \"" << memberName
+					          << "\" not found in JSON:" << std::endl
+					          << jDoc << std::endl << std::endl;
+					print_stacktrace();
+				}
+				ctx.mOk = false;
+				break;
+			}
+
+			rapidjson::Value& v = jDoc[mName];
+
+			RsGenericSerializer::SerializeContext lCtx(nullptr, 0, ctx.mFlags);
+			lCtx.mJson.SetObject() = v; // Beware of move semantic!!
+
+			serial_process(j, ctx, p.first, "first");
+			serial_process(j, ctx, p.second, "second");
+			ctx.mOk &= lCtx.mOk;
+
+			break;
+		}
+		default:
+			std::cerr << __PRETTY_FUNCTION__ << " Unknown serial job: "
+			          << static_cast<std::underlying_type<decltype(j)>::type>(j)
+			          << std::endl;
+			exit(EINVAL);
+			break;
+		}
+	}
+
 	/// std::vector<T>
 	template<typename T>
 	static void serial_process( RsGenericSerializer::SerializeJob j,

libretroshare/src/turtle/p3turtle.cc

@@ -1112,7 +1112,7 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
     std::list<std::pair<RsTurtleSearchResultItem*,RsTurtleClientService*> > results_to_notify_off_mutex ;
 
 	{
-		RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
+		RS_STACK_MUTEX(mTurtleMtx);
 		// Find who actually sent the corresponding request.
 		//
 		std::map<TurtleRequestId,TurtleSearchRequestInfo>::iterator it = _search_requests_origins.find(item->request_id) ;
@@ -1178,7 +1178,7 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
 
 			sendItem(fwd_item) ;
 		}
-	}
+	} // mTurtleMtx end
 
     // now we notify clients off-mutex.