diff --git a/libretroshare/src/file_sharing/file_tree.cc b/libretroshare/src/file_sharing/file_tree.cc index 55de0e78b..3cf893402 100644 --- a/libretroshare/src/file_sharing/file_tree.cc +++ b/libretroshare/src/file_sharing/file_tree.cc @@ -185,6 +185,11 @@ std::unique_ptr RsFileTree::fromDirDetails( ft->mFiles.push_back(fd); ft->mTotalFiles = 1; ft->mTotalSize = fd.size; + + DirData dd; + dd.name = "/"; + dd.subfiles.push_back(0); + ft->mDirs.push_back(dd); } else recurs_buildFileTree(*ft, 0, dd, remote, remove_top_dirs ); return ft; diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index 0af02144f..29494f189 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -38,7 +38,7 @@ #include "util/rsdiscspace.h" #include "util/rsmemory.h" #include "util/rstime.h" - +#include "util/cxx17retrocompat.h" #include "ft/ftcontroller.h" #include "ft/ftfilecreator.h" @@ -923,10 +923,15 @@ bool ftController::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) return false ; } -bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash, - uint64_t size, const std::string& dest, TransferRequestFlags flags, - const std::list &_srcIds, uint16_t state) +bool ftController::FileRequest( + const std::string& fname, const RsFileHash& hash, uint64_t size, + const std::string& dest, TransferRequestFlags flags, + const std::list &_srcIds, uint16_t state ) { + /* TODO: To support collections faithfully we need to be able to save + * the same file to multiple locations, both if already downloaded or + * incomplete */ + std::list srcIds(_srcIds) ; /* check if we have the file */ @@ -965,8 +970,9 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash // create void file with the target name. FILE *f = RsDirUtil::rs_fopen(destination.c_str(),"w") ; - if(f == NULL) - std::cerr << "Could not open file " << destination << " for writting." << std::endl ; + if(!f) + RsErr() << __PRETTY_FUNCTION__ << " Could not write file " + << destination << std::endl; else fclose(f) ; @@ -979,12 +985,13 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash // if(size >= 1024ull*1024ull*((1ull << 32) - 1)) { - std::cerr << "FileRequest Error: unexpected size. This is probably a bug." << std::endl; - std::cerr << " name = " << fname << std::endl ; - std::cerr << " flags = " << flags << std::endl ; - std::cerr << " dest = " << dest << std::endl ; - std::cerr << " size = " << size << std::endl ; - return false ; + RsErr() << __PRETTY_FUNCTION__ + << " unexpected size. This is probably a bug." + << " name = " << fname + << " flags = " << flags + << " dest = " << dest + << " size = " << size << std::endl; + return false; } /* If file transfer is not enabled .... @@ -1004,8 +1011,8 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash } } - // remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files. - // + /* remove the sources from the list, if they don't have clearance for direct + * transfer. This happens only for non cache files. */ for(std::list::iterator it = srcIds.begin(); it != srcIds.end(); ) { bool bAllowDirectDL = false; @@ -1052,11 +1059,10 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash * This is important as some guis request duplicate files regularly. */ - { - RsStackMutex stack(ctrlMutex); /******* LOCKED ********/ - - std::map::const_iterator dit = mDownloads.find(hash); + { + RS_STACK_MUTEX(ctrlMutex); + auto dit = std::as_const(mDownloads).find(hash); if (dit != mDownloads.end()) { /* we already have it! */ @@ -1110,7 +1116,7 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash return true; } - } /******* UNLOCKED ********/ + } // RS_STACK_MUTEX(ctrlMutex); unlocked if(mSearch && !(flags & RS_FILE_REQ_NO_SEARCH)) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 678ac4a02..7ec4251b3 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -295,7 +295,10 @@ bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info) return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info); } -bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list& srcIds) +bool ftServer::FileRequest( + const std::string& fname, const RsFileHash& hash, uint64_t size, + const std::string& dest, TransferRequestFlags flags, + const std::list& srcIds ) { #ifdef SERVER_DEBUG FTSERVER_DEBUG() << "Requesting " << fname << std::endl ; @@ -307,6 +310,93 @@ bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uin return true ; } +std::error_condition ftServer::requestFiles( + const RsFileTree& collection, const std::string& destPath, + const std::vector& srcIds, FileRequestFlags flags ) +{ + constexpr auto fname = __PRETTY_FUNCTION__; + const auto dirsCount = collection.mDirs.size(); + const auto filesCount = collection.mFiles.size(); + + Dbg2() << fname << " dirsCount: " << dirsCount + << " filesCount: " << filesCount << std::endl; + + if(!dirsCount) + { + RsErr() << fname << " Directories list empty in collection " + << std::endl; + return std::errc::not_a_directory; + } + + if(!filesCount) + { + RsErr() << fname << " Files list empty in collection " << std::endl; + return std::errc::invalid_argument; + } + + if(filesCount != collection.mTotalFiles) + { + RsErr() << fname << " Files count mismatch" << std::endl; + return std::errc::invalid_argument; + } + + std::string basePath = destPath.empty() ? getDownloadDirectory() : destPath; + // Track how many time a directory have been explored + std::vector dirsSeenCnt(dirsCount, 0); + // + using StackEntry = std::tuple; + std::deque dStack = { std::make_tuple(0, basePath) }; + + const auto exploreDir = [&](const StackEntry& se)-> std::error_condition + { + std::uintptr_t dirHandle; std::string parentPath; + std::tie(dirHandle, parentPath) = se; + + const auto& dirData = collection.mDirs[dirHandle]; + auto& seenTimes = dirsSeenCnt[dirHandle]; + std::string dirPath = RsDirUtil::makePath(parentPath, dirData.name); + + /* This check is not perfect but is cheap and interrupt loop exploration + * before it becomes pathological */ + if(seenTimes++ > dirsCount) + { + RsErr() << fname << " loop detected! dir: " + << dirHandle << " \"" << dirPath + << "\" explored too many times" << std::endl; + return std::errc::too_many_symbolic_link_levels; + } + + for(auto fHandle: dirData.subfiles) + { + if(fHandle >= filesCount) return std::errc::argument_out_of_domain; + + const RsFileTree::FileData& fData = collection.mFiles[fHandle]; + + bool fr = + FileRequest( fData.name, fData.hash, fData.size, + dirPath, + TransferRequestFlags::fromEFT(flags), + std::list(srcIds.begin(), srcIds.end()) ); + + Dbg2() << fname << " requested: " << fr << " " + << fData.hash << " -> " << dirPath << std::endl; + } + + for(auto dHandle: dirData.subdirs) + dStack.push_back(std::make_tuple(dHandle, dirPath)); + + return std::error_condition(); + }; + + while(!dStack.empty()) + { + if(std::error_condition ec = exploreDir(dStack.front())) return ec; + dStack.pop_front(); + } + + return std::error_condition(); +} + bool ftServer::activateTunnels(const RsFileHash& hash,uint32_t encryption_policy,TransferRequestFlags flags,bool onoff) { RsFileHash hash_of_hash ; diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index f06f0f5bd..7c8ca5ff2 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -206,6 +206,14 @@ public: virtual void setFilePermDirectDL(uint32_t perm) ; virtual uint32_t filePermDirectDL() ; + /// @see RsFiles + std::error_condition requestFiles( + const RsFileTree& collection, + const std::string& destPath = "", + const std::vector& srcIds = std::vector(), + FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING + ) override; + /// @see RsFiles bool turtleSearchRequest( const std::string& matchString, diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index ae5c00dea..661043a20 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -489,10 +489,10 @@ public: /** * @brief Initiate downloading of a file * @jsonapi{development} - * @param[in] fileName - * @param[in] hash - * @param[in] size - * @param[in] destPath in not empty specify a destination path + * @param[in] fileName file name + * @param[in] hash file hash + * @param[in] size file size + * @param[in] destPath optional specify the destination directory * @param[in] flags you usually want RS_FILE_REQ_ANONYMOUS_ROUTING * @param[in] srcIds eventually specify known sources * @return false if we already have the file, true otherwhise @@ -502,6 +502,25 @@ public: const std::string& destPath, TransferRequestFlags flags, const std::list& srcIds ) = 0; + /** + * @brief Initiate download of a files collection + * @jsonapi{development} + * An usually useful companion method of this is @see parseFilesLink() + * @param[in] collection collection of files to download + * @param[in] destPath optional base path on which to download the + * collection, if left empty the default download directory will be used + * @param[in] srcIds optional peers id known as direct source of the + * collection + * @param[in] flags optional flags to fine tune search and download + * algorithm + * @return success or error details. + */ + virtual std::error_condition requestFiles( + const RsFileTree& collection, + const std::string& destPath = "", + const std::vector& srcIds = std::vector(), + FileRequestFlags flags = FileRequestFlags::ANONYMOUS_ROUTING ) = 0; + /** * @brief Cancel file downloading * @jsonapi{development} @@ -924,8 +943,6 @@ public: virtual std::error_condition parseFilesLink( const std::string& link, RsFileTree& collection ) = 0; - // virtual std::error_condition downloadFilesLink(FileRequestFlags) - /** * @brief Get list of ignored file name prefixes and suffixes * @param[out] ignoredPrefixes storage for ingored prefixes diff --git a/libretroshare/src/retroshare/rstypes.h b/libretroshare/src/retroshare/rstypes.h index 1f5107fe3..978934942 100644 --- a/libretroshare/src/retroshare/rstypes.h +++ b/libretroshare/src/retroshare/rstypes.h @@ -366,6 +366,8 @@ struct DirDetails : RsSerializable RS_SERIAL_PROCESS(children); RS_SERIAL_PROCESS(parent_groups); } + + ~DirDetails() override = default; }; class FileDetail diff --git a/libretroshare/src/util/rsdir.cc b/libretroshare/src/util/rsdir.cc index 1b794f982..be327bb3d 100644 --- a/libretroshare/src/util/rsdir.cc +++ b/libretroshare/src/util/rsdir.cc @@ -265,36 +265,48 @@ bool RsDirUtil::fileExists(const std::string& filename) bool RsDirUtil::moveFile(const std::string& source,const std::string& dest) { - // Check that the destination directory exists. If not, create it. + Dbg3() << __PRETTY_FUNCTION__<< " source: " << source + << " dest: " << dest << std::endl; std::string dest_dir ; std::string dest_file ; - splitDirFromFile(dest,dest_dir,dest_file) ; + splitDirFromFile(dest, dest_dir, dest_file); - std::cerr << "Moving file " << source << " to " << dest << std::endl; - std::cerr << "Checking that directory " << dest_dir << " actually exists." << std::endl; + if(!checkDirectory(dest_dir)) + { + if(!std::filesystem::create_directories(dest_dir)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure creating directory: " + << dest_dir << std::endl; + return false; + } + } - if(!checkCreateDirectory(dest_dir)) - return false ; + // First try a rename + if(renameFile(source,dest)) + { + Dbg3() << __PRETTY_FUNCTION__ << " plain rename worked" << std::endl; + return true; + } - // First try a rename - // + /* If not, try to copy. The src and dest probably belong to different file + * systems */ + if(!copyFile(source,dest)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure copying file" << std::endl; + return false; + } - if(renameFile(source,dest)) - return true ; + // delete the original + if(!removeFile(source)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure deleting original file" + << std::endl; + return false; + } - // If not, try to copy. The src and dest probably belong to different file systems - - if(!copyFile(source,dest)) - return false ; - - // copy was successful, let's delete the original - - if(!removeFile(source)) - return false ; - - return true ; + return true; } bool RsDirUtil::removeFile(const std::string& filename) @@ -425,7 +437,7 @@ bool RsDirUtil::checkFile(const std::string& filename,uint64_t& file_size,bool d } -bool RsDirUtil::checkDirectory(const std::string& dir) +bool RsDirUtil::checkDirectory(const std::string& dir) { int val; mode_t st_mode; @@ -461,11 +473,9 @@ bool RsDirUtil::checkDirectory(const std::string& dir) } -bool RsDirUtil::checkCreateDirectory(const std::string& dir) +bool RsDirUtil::checkCreateDirectory(const std::string& dir) { -#ifdef RSDIR_DEBUG - std::cerr << "RsDirUtil::checkCreateDirectory() dir: " << dir << std::endl; -#endif + Dbg3() << __PRETTY_FUNCTION__ << " " << dir << std::endl; #ifdef WINDOWS_SYS std::wstring wdir; @@ -516,6 +526,23 @@ bool RsDirUtil::checkCreateDirectory(const std::string& dir) return true; } +#if __cplusplus < 201703L +bool std::filesystem::create_directories(const std::string& path) +{ + for( std::string::size_type lastIndex = 0; lastIndex < std::string::npos; + lastIndex = path.find('/', lastIndex) ) + { + std::string&& curDir = path.substr(0, ++lastIndex); + if(!RsDirUtil::checkCreateDirectory(curDir)) + { + RsErr() << __PRETTY_FUNCTION__ << " failure creating: " << curDir + << " of: " << path << std::endl; + return false; + } + } + return true; +} +#endif // __cplusplus < 201703L std::string RsDirUtil::removeSymLinks(const std::string& path) { diff --git a/libretroshare/src/util/rsdir.h b/libretroshare/src/util/rsdir.h index b6ef4891d..e8ff09943 100644 --- a/libretroshare/src/util/rsdir.h +++ b/libretroshare/src/util/rsdir.h @@ -84,7 +84,10 @@ int breakupDirList(const std::string& path, std::list &subdirs bool splitDirFromFile(const std::string& full_path,std::string& dir, std::string& file); bool copyFile(const std::string& source,const std::string& dest); -bool moveFile(const std::string& source,const std::string& dest); + +/** Move file. If destination directory doesn't exists create it. */ +bool moveFile(const std::string& source, const std::string& dest); + bool removeFile(const std::string& file); bool fileExists(const std::string& file); bool checkFile(const std::string& filename,uint64_t& file_size,bool disallow_empty_file = false); @@ -141,8 +144,23 @@ bool getWideFileHash(std::wstring filepath, RsFileHash &hash, u FILE *rs_fopen(const char* filename, const char* mode); std::string convertPathToUnix(std::string path); + +/** Concatenate two path pieces putting '/' separator between them only if + * needed */ std::string makePath(const std::string &path1, const std::string &path2); + +RS_SET_CONTEXT_DEBUG_LEVEL(1); } - +#if __cplusplus < 201703L +namespace std +{ +namespace filesystem +{ +bool create_directories(const std::string& path); +} +} +#endif // __cplusplus < 201703L + + #endif