added method in ftServer to allow retrieving data from shared/downloaded files (patch from electron, modified)

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@7999 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/ft/ftchunkmap.cc

@@ -402,7 +402,17 @@ void ChunkMap::removeInactiveChunks(std::vector<ftChunk::ChunkId>& to_remove)
 			++it ;
 }
 
-bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const 
+bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const
+{
+    return hasChunkState(offset, chunk_size, FileChunksInfo::CHUNK_DONE);
+}
+
+bool ChunkMap::isChunkOutstanding(uint64_t offset, uint32_t chunk_size) const
+{
+    return hasChunkState(offset, chunk_size, FileChunksInfo::CHUNK_OUTSTANDING);
+}
+
+bool ChunkMap::hasChunkState(uint64_t offset, uint32_t chunk_size, FileChunksInfo::ChunkState state) const
 {
 	uint32_t chunk_number_start = offset/(uint64_t)_chunk_size ;
 	uint32_t chunk_number_end = (offset+(uint64_t)chunk_size)/(uint64_t)_chunk_size ;
@@ -414,16 +424,16 @@ bool ChunkMap::isChunkAvailable(uint64_t offset, uint32_t chunk_size) const
 	// chunk_size=0, and offset%_chunk_size=0, so the response "true" is still valid.
 	//
 	for(uint32_t i=chunk_number_start;i<chunk_number_end;++i)
-		if(_map[i] != FileChunksInfo::CHUNK_DONE)
+        if(_map[i] != state)
 		{
 #ifdef DEBUG_FTCHUNK
-			std::cerr << "ChunkMap::isChunkAvailable(): (" << offset << "," << chunk_size << ") is not available" << std::endl; 
+            std::cerr << "ChunkMap::hasChunkState(): (" << offset << "," << chunk_size << ") has different state" << std::endl;
 #endif
 			return false ;
 		}
 
 #ifdef DEBUG_FTCHUNK
-	std::cerr << "ChunkMap::isChunkAvailable(): (" << offset << "," << chunk_size << ") is available" << std::endl; 
+    std::cerr << "ChunkMap::hasChunkState(): (" << offset << "," << chunk_size << ") check returns true" << std::endl;
 #endif
 	return true ;
 }

libretroshare/src/ft/ftchunkmap.h

@@ -159,6 +159,8 @@ class ChunkMap
 		/// This function is used by the parent ftFileProvider to know whether the chunk can be sent or not.
 		bool isChunkAvailable(uint64_t offset, uint32_t chunk_size) const ;
 
+        bool isChunkOutstanding(uint64_t offset, uint32_t chunk_size) const ;
+
 		/// Remove active chunks that have not received any data for the last 60 seconds, and return
 		/// the list of slice numbers that should be canceled.
 		void removeInactiveChunks(std::vector<ftChunk::ChunkId>& to_remove) ;
@@ -206,6 +208,8 @@ class ChunkMap
 		uint32_t getAvailableChunk(const RsPeerId& peer_id,bool& chunk_map_too_old) ;
 
 	private:
+        bool hasChunkState(uint64_t offset, uint32_t chunk_size, FileChunksInfo::ChunkState state) const;
+
 		uint64_t												_file_size ;						//! total size of the file in bytes.
 		uint32_t												_chunk_size ;						//! Size of chunks. Common to all chunks.
 		FileChunksInfo::ChunkStrategy 				_strategy ;							//! how do we allocate new chunks

libretroshare/src/ft/ftcontroller.cc

@@ -1524,7 +1524,7 @@ bool 	ftController::FileClearCompleted()
 }
 
 	/* get Details of File Transfers */
-bool 	ftController::FileDownloads(std::list<RsFileHash> &hashs)
+void 	ftController::FileDownloads(std::list<RsFileHash> &hashs)
 {
 	RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
 
@@ -1537,7 +1537,6 @@ bool 	ftController::FileDownloads(std::list<RsFileHash> &hashs)
 	{
 		hashs.push_back(it->second->mHash);
 	}
-	return true;
 }
 
 

libretroshare/src/ft/ftcontroller.h

@@ -167,7 +167,7 @@ class ftController: public CacheTransfer, public RsThread, public pqiServiceMoni
 		uint32_t getMinPrioritizedTransfers() ;
 
 		/* get Details of File Transfers */
-        bool 	FileDownloads(std::list<RsFileHash> &hashs);
+        void FileDownloads(std::list<RsFileHash> &hashs);
 
 		/* Directory Handling */
         bool 	setDownloadDirectory(std::string path);

libretroshare/src/ft/ftdatamultiplex.cc

@@ -77,6 +77,42 @@ ftDataMultiplex::ftDataMultiplex(const RsPeerId& ownId, ftDataSend *server, ftSe
 	return;
 }
 
+bool ftDataMultiplex::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size, uint8_t *data)
+{
+    RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
+    ftFileProvider* provider = 0;
+
+    std::map<RsFileHash, ftClient>::iterator cit;
+    std::map<RsFileHash, ftFileProvider *>::iterator sit;
+
+    // check if file is currently downloading
+    if (mClients.end() != (cit = mClients.find(hash)))
+        provider = (cit->second).mCreator;
+
+    // else check if its already uploading
+    else if (mServers.end() != (sit = mServers.find(hash)))
+        provider = sit->second;
+
+    // else create a new provider
+    else
+    {
+        FileInfo info;
+        FileSearchFlags hintflags =   RS_FILE_HINTS_EXTRA | RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_SPEC_ONLY | RS_FILE_HINTS_NETWORK_WIDE;
+        if(mSearch->search(hash, hintflags, info))
+        {
+            provider = new ftFileProvider(info.path, info.size, hash);
+            mServers[hash] = provider;
+        }
+    }
+
+    if(!provider || ! provider->getFileData(mOwnId, offset, requested_size, data, true))
+    {
+        requested_size = 0 ;
+        return false ;
+    }
+    return true ;
+}
+
 bool	ftDataMultiplex::addTransferModule(ftTransferModule *mod, ftFileCreator *f)
 {
 	RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/

libretroshare/src/ft/ftdatamultiplex.h

@@ -95,6 +95,16 @@ class ftDataMultiplex: public ftDataRecv, public RsQueueThread
 
 		ftDataMultiplex(const RsPeerId& ownId, ftDataSend *server, ftSearch *search);
 
+        /**
+         * @see RsFiles::getFileData
+     *
+     * data should be pre-allocated by the client with size sufficient gfor requested_size bytes.
+     * requested_size will be changed so as to contain the actual number of bytes copied from the file,
+     * in case where the full size wasn't available.
+     * False is returned if no data can be obtained from that file.
+         */
+        bool getFileData(const RsFileHash& hash, uint64_t offset,uint32_t& requested_size, uint8_t *data);
+
 		/* ftController Interface */
 		bool	addTransferModule(ftTransferModule *mod, ftFileCreator *f);
 		bool	removeTransferModule(const RsFileHash& hash);

libretroshare/src/ft/ftfilecreator.cc

@@ -65,7 +65,7 @@ ftFileCreator::ftFileCreator(const std::string& path, uint64_t size, const RsFil
 #endif
 }
 
-bool ftFileCreator::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data)
+bool ftFileCreator::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool allow_unverified)
 {
 	// Only send the data if we actually have it.
 	//
@@ -77,6 +77,37 @@ bool ftFileCreator::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_
 		RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
 
 		have_it = chunkMap.isChunkAvailable(offset, chunk_size) ;
+
+#define ENABLE_SLICES
+#ifdef ENABLE_SLICES
+        // try if we have data from an incomplete or not veryfied chunk
+        if(!have_it && allow_unverified)
+        {
+            std::map<uint64_t, ftChunk>::iterator it;
+            have_it = true;
+            // this map contains chunks which are currently being downloaded
+            for(std::map<uint64_t,ftChunk>::iterator it=mChunks.begin(); it!=mChunks.end(); ++it)
+            {
+                ftChunk chunk = it->second;
+                // begin of slice is in requested range
+                if(chunk.offset >= offset && chunk.offset < (offset+chunk_size))
+                {
+                    // reduce the requested size
+                    chunk_size = chunk.offset - offset;
+                }
+                // end of slice is in requested range
+                if((chunk.offset+chunk.size) >= offset && (chunk.offset+chunk.size) < (offset+chunk_size))
+                {
+                    // can do nothing about this
+                    have_it = false;
+                }
+            }
+            // check if the chunk was already started to download
+            // if not, we don't have it
+            if(chunkMap.isChunkOutstanding(offset, chunk_size))
+                have_it = false;
+        }
+#endif
 	}
 #ifdef FILE_DEBUG
 	if(have_it)

libretroshare/src/ft/ftfilecreator.h

@@ -52,7 +52,7 @@ class ftFileCreator: public ftFileProvider
 		~ftFileCreator();
 
 		/* overloaded from FileProvider */
-		virtual bool 	getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data);
+        virtual bool 	getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool allow_unverified = false);
 		bool	finished() ;
 		uint64_t getRecvd();
 

libretroshare/src/ft/ftfileprovider.cc

@@ -139,7 +139,7 @@ void ftFileProvider::getAvailabilityMap(CompressedChunkMap& cmap)
 }
 
 
-bool ftFileProvider::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data)
+bool ftFileProvider::getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool /*allow_unverified*/)
 {
 	/* dodgey checking outside of mutex...
 	 * much check again inside FileAttrs().

libretroshare/src/ft/ftfileprovider.h

@@ -41,7 +41,16 @@ class ftFileProvider
 		ftFileProvider(const std::string& path, uint64_t size, const RsFileHash& hash);
 		virtual ~ftFileProvider();
 
-		virtual bool 	getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data);
+        /**
+         * read a block of data from the file
+         * @param peer_id for the uploading stats: to which peer the data will be send
+         * @param offset begin of the requested data range
+         * @param chunk_size how many bytes to read. Will be set to the number of valid bytes in data on return.
+         * @param data pointer to a buffer of size chunk_size. Contains the data on success.
+         * @param allow_unverified for ftFileCreator: set to true to return data from unverified chunks. Use it if you want data asap.
+         * @return true if data was read
+         */
+        virtual bool 	getFileData(const RsPeerId& peer_id,uint64_t offset, uint32_t &chunk_size, void *data, bool allow_unverified = false);
 		virtual bool    FileDetails(FileInfo &info);
 		RsFileHash getHash();
 		uint64_t getFileSize();

libretroshare/src/ft/ftserver.cc

@@ -262,6 +262,11 @@ bool ftServer::checkHash(const RsFileHash& hash,std::string& error_string)
     return true ;
 }
 
+bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data)
+{
+    return mFtDataplex->getFileData(hash, offset, requested_size,data);
+}
+
 bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
 {
 	return mFtController->alreadyHaveFile(hash, info);
@@ -403,9 +408,9 @@ std::string ftServer::getPartialsDirectory()
 	/************************* Other Access ************************/
 	/***************************************************************/
 
-bool ftServer::FileDownloads(std::list<RsFileHash> &hashs)
+void ftServer::FileDownloads(std::list<RsFileHash> &hashs)
 {
-	return mFtController->FileDownloads(hashs);
+    mFtController->FileDownloads(hashs);
 	/* this only contains downloads.... not completed */
 	//return mFtDataplex->FileDownloads(hashs);
 }

libretroshare/src/ft/ftserver.h

@@ -124,6 +124,11 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
 		ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; }
 		ftController *getController() const { return mFtController ; }
 
+        /**
+         * @see RsFiles::getFileData
+         */
+        bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data);
+
 		/***
 		 * Control of Downloads
 		 ***/
@@ -157,7 +162,7 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
 		/***
 		 * Download/Upload Details
 		 ***/
-		virtual bool FileDownloads(std::list<RsFileHash> &hashs);
+        virtual void FileDownloads(std::list<RsFileHash> &hashs);
 		virtual bool FileUploads(std::list<RsFileHash> &hashs);
 		virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info);
 		virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ;

libretroshare/src/retroshare/rsfiles.h

@@ -109,9 +109,22 @@ class RsFiles
 		RsFiles() { return; }
 		virtual ~RsFiles() { return; }
 
-		/****************************************/
+        /**
-		/* download */
+         * Provides file data for the gui: media streaming or rpc clients.
-
+         * It may return unverified chunks. This allows streaming without having to wait for hashes or completion of the file.
+         * This function returns an unspecified amount of bytes. Either as much data as available or a sensible maximum. Expect a block size of around 1MiB.
+         * To get more data, call this function repeatedly with different offsets.
+         * Returns false in case
+         * - the files is not available on the local node
+         * - not downloading
+         * - the requested data was not downloaded yet
+         * - end of file was reached
+         * @param hash hash of a file. The file has to be available on this node or it has to be in downloading state.
+         * @param offset where the desired block starts
+     * @param requested_size size of pre-allocated data. Will be updated by the function.
+         * @param data pre-allocated memory chunk of size 'requested_size' by the client
+         */
+        virtual bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data)=0;
 
 		/***
 		 *  Control of Downloads.
@@ -147,7 +160,7 @@ class RsFiles
 		/***
 		 * Download / Upload Details.
 		 ***/
-		virtual bool FileDownloads(std::list<RsFileHash> &hashs) = 0;
+        virtual void FileDownloads(std::list<RsFileHash> &hashs) = 0;
 		virtual bool FileUploads(std::list<RsFileHash> &hashs) = 0;
 		virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) = 0;