Add RsFiles::requestFiles to de API to download whole colletions

Fix filetree creation from single file
RsDirUtil::moveFile now works also if parent directories doesn't exists
Backport std::filesystem::create_directories from C++17

libretroshare/src/file_sharing/file_tree.cc

@@ -185,6 +185,11 @@ std::unique_ptr<RsFileTree> RsFileTree::fromDirDetails(
 		ft->mFiles.push_back(fd);
 		ft->mTotalFiles = 1;
 		ft->mTotalSize = fd.size;
+
+		DirData dd;
+		dd.name = "/";
+		dd.subfiles.push_back(0);
+		ft->mDirs.push_back(dd);
 	}
 	else recurs_buildFileTree(*ft, 0, dd, remote, remove_top_dirs );
 	return ft;

libretroshare/src/ft/ftcontroller.cc

@@ -38,7 +38,7 @@
 #include "util/rsdiscspace.h"
 #include "util/rsmemory.h"
 #include "util/rstime.h"
-
+#include "util/cxx17retrocompat.h"
 #include "ft/ftcontroller.h"
 
 #include "ft/ftfilecreator.h"
@@ -923,10 +923,15 @@ bool ftController::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
 	return false ;
 }
 
-bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash,
+bool ftController::FileRequest(
-			uint64_t size, const std::string& dest, TransferRequestFlags flags,
+        const std::string& fname, const RsFileHash& hash, uint64_t size,
-                                                                const std::list<RsPeerId> &_srcIds, uint16_t state)
+        const std::string& dest, TransferRequestFlags flags,
+        const std::list<RsPeerId> &_srcIds, uint16_t state )
 {
+	/* TODO: To support collections faithfully we need to be able to save
+	 * the same file to multiple locations, both if already downloaded or
+	 * incomplete */
+
     std::list<RsPeerId> srcIds(_srcIds) ;
 
 	/* check if we have the file */
@@ -965,8 +970,9 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 
 		// create void file with the target name.
 		FILE *f = RsDirUtil::rs_fopen(destination.c_str(),"w") ;
-		if(f == NULL)
+		if(!f)
-			std::cerr << "Could not open file " << destination << " for writting." << std::endl ;
+			RsErr() << __PRETTY_FUNCTION__ << " Could not write file "
+			        << destination << std::endl;
 		else
 			fclose(f) ;
 
@@ -979,12 +985,13 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 	//
 	if(size >= 1024ull*1024ull*((1ull << 32) - 1))
 	{
-		std::cerr << "FileRequest Error: unexpected size. This is probably a bug." << std::endl;
+		RsErr() << __PRETTY_FUNCTION__
-		std::cerr << "  name  = " << fname << std::endl ;
+		        << " unexpected size. This is probably a bug."
-		std::cerr << "  flags = " << flags << std::endl ;
+		        << " name  = " << fname
-		std::cerr << "  dest  = " << dest << std::endl ;
+		        << " flags = " << flags
-		std::cerr << "  size  = " << size << std::endl ;
+		        << " dest  = " << dest
-		return false ;
+		        << " size  = " << size << std::endl;
+		return false;
 	}
 
 	/* If file transfer is not enabled ....
@@ -1004,8 +1011,8 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 		}
 	}
 
-	// remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files.
+	/* remove the sources from the list, if they don't have clearance for direct
-	//
+	 * transfer. This happens only for non cache files. */
 	for(std::list<RsPeerId>::iterator it = srcIds.begin(); it != srcIds.end(); )
 	{
 		bool bAllowDirectDL = false;
@@ -1052,11 +1059,10 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 	 * This is important as some guis request duplicate files regularly.
 	 */
 
-	{ 
+	{
-		RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
+		RS_STACK_MUTEX(ctrlMutex);
-
-        std::map<RsFileHash, ftFileControl*>::const_iterator dit = mDownloads.find(hash);
 
+		auto dit = std::as_const(mDownloads).find(hash);
 		if (dit != mDownloads.end())
 		{
 			/* we already have it! */
@@ -1110,7 +1116,7 @@ bool 	ftController::FileRequest(const std::string& fname, const RsFileHash& hash
 
 			return true;
 		}
-	} /******* UNLOCKED ********/
+	} // RS_STACK_MUTEX(ctrlMutex); unlocked
 
 
 	if(mSearch && !(flags & RS_FILE_REQ_NO_SEARCH))

libretroshare/src/ft/ftserver.cc

@@ -295,7 +295,10 @@ bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
 	return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info);
 }
 
-bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list<RsPeerId>& srcIds)
+bool ftServer::FileRequest(
+        const std::string& fname, const RsFileHash& hash, uint64_t size,
+        const std::string& dest, TransferRequestFlags flags,
+        const std::list<RsPeerId>& srcIds )
 {
 #ifdef SERVER_DEBUG
 	FTSERVER_DEBUG() << "Requesting " << fname << std::endl ;
@@ -307,6 +310,93 @@ bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uin
 	return true ;
 }
 
+std::error_condition ftServer::requestFiles(
+        const RsFileTree& collection, const std::string& destPath,
+        const std::vector<RsPeerId>& srcIds, FileRequestFlags flags )
+{
+	constexpr auto fname = __PRETTY_FUNCTION__;
+	const auto dirsCount = collection.mDirs.size();
+	const auto filesCount = collection.mFiles.size();
+
+	Dbg2() << fname << " dirsCount: " << dirsCount
+	       << " filesCount: " << filesCount << std::endl;
+
+	if(!dirsCount)
+	{
+		RsErr() << fname << " Directories list empty in collection "
+		        << std::endl;
+		return std::errc::not_a_directory;
+	}
+
+	if(!filesCount)
+	{
+		RsErr() << fname << " Files list empty in collection " << std::endl;
+		return std::errc::invalid_argument;
+	}
+
+	if(filesCount != collection.mTotalFiles)
+	{
+		RsErr() << fname << " Files count mismatch" << std::endl;
+		return std::errc::invalid_argument;
+	}
+
+	std::string basePath = destPath.empty() ? getDownloadDirectory() : destPath;
+	// Track how many time a directory have been explored
+	std::vector<uint32_t> dirsSeenCnt(dirsCount, 0);
+	//                          <directory handle, parent path>
+	using StackEntry = std::tuple<std::uintptr_t, std::string>;
+	std::deque<StackEntry> dStack = { std::make_tuple(0, basePath) };
+
+	const auto exploreDir = [&](const StackEntry& se)-> std::error_condition
+	{
+		std::uintptr_t dirHandle; std::string parentPath;
+		std::tie(dirHandle, parentPath) = se;
+
+		const auto& dirData = collection.mDirs[dirHandle];
+		auto& seenTimes = dirsSeenCnt[dirHandle];
+		std::string dirPath = RsDirUtil::makePath(parentPath, dirData.name);
+
+		/* This check is not perfect but is cheap and interrupt loop exploration
+		 * before it becomes pathological */
+		if(seenTimes++ > dirsCount)
+		{
+			RsErr() << fname << " loop detected! dir: "
+			        << dirHandle << " \"" << dirPath
+			        << "\" explored too many times" << std::endl;
+			return std::errc::too_many_symbolic_link_levels;
+		}
+
+		for(auto fHandle: dirData.subfiles)
+		{
+			if(fHandle >= filesCount) return std::errc::argument_out_of_domain;
+
+			const RsFileTree::FileData& fData = collection.mFiles[fHandle];
+
+			bool fr =
+			FileRequest( fData.name, fData.hash, fData.size,
+			             dirPath,
+			             TransferRequestFlags::fromEFT<FileRequestFlags>(flags),
+			             std::list<RsPeerId>(srcIds.begin(), srcIds.end()) );
+
+			Dbg2() << fname << " requested: " << fr << " "
+			       << fData.hash << " -> " << dirPath << std::endl;
+		}
+
+		for(auto dHandle: dirData.subdirs)
+			dStack.push_back(std::make_tuple(dHandle, dirPath));
+
+		return std::error_condition();
+	};
+
+	while(!dStack.empty())
+	{
+		if(std::error_condition ec = exploreDir(dStack.front())) return ec;
+		dStack.pop_front();
+	}
+
+	return std::error_condition();
+}
+
 bool ftServer::activateTunnels(const RsFileHash& hash,uint32_t encryption_policy,TransferRequestFlags flags,bool onoff)
 {
 	RsFileHash hash_of_hash ;

libretroshare/src/ft/ftserver.h

@@ -206,6 +206,14 @@ public:
 	virtual void setFilePermDirectDL(uint32_t perm) ;
 	virtual uint32_t filePermDirectDL() ;
 
+	/// @see RsFiles
+	std::error_condition requestFiles(
+	        const RsFileTree& collection,
+	        const std::string& destPath = "",
+	        const std::vector<RsPeerId>& srcIds = std::vector<RsPeerId>(),
+	        FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING
+	        ) override;
+
 	/// @see RsFiles
 	bool turtleSearchRequest(
 	        const std::string& matchString,

libretroshare/src/retroshare/rsfiles.h

@@ -489,10 +489,10 @@ public:
 	/**
 	 * @brief Initiate downloading of a file
 	 * @jsonapi{development}
-	 * @param[in] fileName
+	 * @param[in] fileName file name
-	 * @param[in] hash
+	 * @param[in] hash file hash
-	 * @param[in] size
+	 * @param[in] size file size
-	 * @param[in] destPath in not empty specify a destination path
+	 * @param[in] destPath optional specify the destination directory
 	 * @param[in] flags you usually want RS_FILE_REQ_ANONYMOUS_ROUTING
 	 * @param[in] srcIds eventually specify known sources
 	 * @return false if we already have the file, true otherwhise
@@ -502,6 +502,25 @@ public:
 	        const std::string& destPath, TransferRequestFlags flags,
 	        const std::list<RsPeerId>& srcIds ) = 0;
 
+	/**
+	 * @brief Initiate download of a files collection
+	 * @jsonapi{development}
+	 * An usually useful companion method of this is @see parseFilesLink()
+	 * @param[in] collection collection of files to download
+	 * @param[in] destPath optional base path on which to download the
+	 *	collection, if left empty the default download directory will be used
+	 * @param[in] srcIds optional peers id known as direct source of the
+	 *	collection
+	 * @param[in] flags optional flags to fine tune search and download
+	 *	algorithm
+	 * @return success or error details.
+	 */
+	virtual std::error_condition requestFiles(
+	        const RsFileTree& collection,
+	        const std::string& destPath = "",
+	        const std::vector<RsPeerId>& srcIds = std::vector<RsPeerId>(),
+	        FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING ) = 0;
+
 	/**
 	 * @brief Cancel file downloading
 	 * @jsonapi{development}
@@ -924,8 +943,6 @@ public:
 	virtual std::error_condition parseFilesLink(
 	        const std::string& link, RsFileTree& collection ) = 0;
 
-	// virtual std::error_condition downloadFilesLink(FileRequestFlags)
-
 	/**
 	 * @brief Get list of ignored file name prefixes and suffixes
 	 * @param[out] ignoredPrefixes storage for ingored prefixes

libretroshare/src/retroshare/rstypes.h

@@ -366,6 +366,8 @@ struct DirDetails : RsSerializable
 		RS_SERIAL_PROCESS(children);
 		RS_SERIAL_PROCESS(parent_groups);
 	}
+
+	~DirDetails() override = default;
 };
 
 class FileDetail

libretroshare/src/util/rsdir.cc

@@ -265,36 +265,48 @@ bool RsDirUtil::fileExists(const std::string& filename)
 
 bool RsDirUtil::moveFile(const std::string& source,const std::string& dest)
 {
-	// Check that the destination directory exists. If not, create it.
+	Dbg3() << __PRETTY_FUNCTION__<< " source: " << source
+	       << " dest: " << dest << std::endl;
 
 	std::string dest_dir ;
 	std::string dest_file ;
 
-	splitDirFromFile(dest,dest_dir,dest_file) ;
+	splitDirFromFile(dest, dest_dir, dest_file);
 
-	std::cerr << "Moving file " << source << " to " << dest << std::endl;
+	if(!checkDirectory(dest_dir))
-	std::cerr << "Checking that directory " << dest_dir << " actually exists." << std::endl;
+	{
+		if(!std::filesystem::create_directories(dest_dir))
+		{
+			RsErr() << __PRETTY_FUNCTION__ << " failure creating directory: "
+			        << dest_dir << std::endl;
+			return false;
+		}
+	}
 
-	if(!checkCreateDirectory(dest_dir))
+	// First try a rename
-		return false ;
+	if(renameFile(source,dest))
+	{
+		Dbg3() << __PRETTY_FUNCTION__ << " plain rename worked" << std::endl;
+		return true;
+	}
 
-    // First try a rename
+	/* If not, try to copy. The src and dest probably belong to different file
-	//
+	 * systems */
+	if(!copyFile(source,dest))
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " failure copying file" << std::endl;
+		return false;
+	}
 
-    if(renameFile(source,dest))
+	// delete the original
-        return true ;
+	if(!removeFile(source))
+	{
+		RsErr() << __PRETTY_FUNCTION__ << " failure deleting original file"
+		        << std::endl;
+		return false;
+	}
 
-    // If not, try to copy. The src and dest probably belong to different file systems
+	return true;
-
-    if(!copyFile(source,dest))
-        return false ;
-
-	// copy was successful, let's delete the original
-
-    if(!removeFile(source))
-        return false ;
-
-    return true ;
 }
 
 bool RsDirUtil::removeFile(const std::string& filename)
@@ -425,7 +437,7 @@ bool	RsDirUtil::checkFile(const std::string& filename,uint64_t& file_size,bool d
 }
 
 
-bool	RsDirUtil::checkDirectory(const std::string& dir)
+bool RsDirUtil::checkDirectory(const std::string& dir)
 {
 	int val;
 	mode_t st_mode;
@@ -461,11 +473,9 @@ bool	RsDirUtil::checkDirectory(const std::string& dir)
 }
 
 
-bool	RsDirUtil::checkCreateDirectory(const std::string& dir)
+bool RsDirUtil::checkCreateDirectory(const std::string& dir)
 {
-#ifdef RSDIR_DEBUG
+	Dbg3() << __PRETTY_FUNCTION__ << " " << dir << std::endl;
-	std::cerr << "RsDirUtil::checkCreateDirectory() dir: " << dir << std::endl;
-#endif
 
 #ifdef WINDOWS_SYS
 	std::wstring wdir;
@@ -516,6 +526,23 @@ bool	RsDirUtil::checkCreateDirectory(const std::string& dir)
 	return true;
 }
 
+#if __cplusplus < 201703L
+bool std::filesystem::create_directories(const std::string& path)
+{
+	for( std::string::size_type lastIndex = 0; lastIndex < std::string::npos;
+	     lastIndex = path.find('/', lastIndex) )
+	{
+		std::string&& curDir = path.substr(0, ++lastIndex);
+		if(!RsDirUtil::checkCreateDirectory(curDir))
+		{
+			RsErr() << __PRETTY_FUNCTION__ << " failure creating: " << curDir
+			        << " of: " << path << std::endl;
+			return false;
+		}
+	}
+	return true;
+}
+#endif // __cplusplus < 201703L
 
 std::string RsDirUtil::removeSymLinks(const std::string& path)
 {

libretroshare/src/util/rsdir.h

@@ -84,7 +84,10 @@ int     	breakupDirList(const std::string& path, std::list<std::string> &subdirs
 bool        splitDirFromFile(const std::string& full_path,std::string& dir, std::string& file);
 
 bool 		copyFile(const std::string& source,const std::string& dest);
-bool 		moveFile(const std::string& source,const std::string& dest);
+
+/** Move file. If destination directory doesn't exists create it. */
+bool moveFile(const std::string& source, const std::string& dest);
+
 bool 		removeFile(const std::string& file);
 bool 		fileExists(const std::string& file);
 bool    	checkFile(const std::string& filename,uint64_t& file_size,bool disallow_empty_file = false);
@@ -141,8 +144,23 @@ bool 		getWideFileHash(std::wstring filepath,                RsFileHash &hash, u
 FILE		*rs_fopen(const char* filename, const char* mode);
 
 std::string convertPathToUnix(std::string path);
+
+/** Concatenate two path pieces putting '/' separator between them only if
+ * needed */
 std::string makePath(const std::string &path1, const std::string &path2);
+
+RS_SET_CONTEXT_DEBUG_LEVEL(1);
 }
 
-	
+#if __cplusplus < 201703L
+namespace std
+{
+namespace filesystem
+{
+bool create_directories(const std::string& path);
+}
+}
+#endif // __cplusplus < 201703L
+
+
 #endif