diff --git a/libretroshare/src/ft/ftfilemapper.cc b/libretroshare/src/ft/ftfilemapper.cc new file mode 100644 index 000000000..1e04582a0 --- /dev/null +++ b/libretroshare/src/ft/ftfilemapper.cc @@ -0,0 +1,301 @@ +#include +#include +#include +#include +#include +#include "ftfilemapper.h" + +//#define DEBUG_FILEMAPPER + +ftFileMapper::ftFileMapper(uint64_t file_size,uint32_t chunk_size) + : _file_size(file_size),_chunk_size(chunk_size) +{ + int nb_chunks = (int)(file_size / (uint64_t)chunk_size) + ( (file_size % chunk_size)==0 ?0:1 ) ; + +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) Creating ftFileMapper for file of size " << file_size << ", with " << nb_chunks << " chunks." << std::endl; +#endif + + _first_free_chunk = 0 ; + _mapped_chunks.clear() ; + _mapped_chunks.resize(nb_chunks,-1) ; + _data_chunks.clear() ; + _data_chunks.resize(nb_chunks,-1) ; +} + +bool ftFileMapper::computeStorageOffset(uint64_t offset,uint64_t& storage_offset) const +{ + // Compute the chunk number for this offset + // + uint32_t cid = (uint32_t)(offset / (uint64_t)_chunk_size) ; + + // Check that the cid is in the allowed range. That should always be the case. + // + if(cid < _mapped_chunks.size() && _mapped_chunks[cid] >= 0) + { + storage_offset = _mapped_chunks[cid]*_chunk_size + (offset % (uint64_t)_chunk_size) ; + return true ; + } + else + { +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) ftFileMapper::computeStorageOffset(): offset " << offset << " corresponds to chunk number " << cid << " which is not mapped!!" << std::endl; +#endif + return false ; + } +} + +bool ftFileMapper::writeData(uint64_t offset,uint32_t size,void *data,FILE *fd) const +{ + if (0 != fseeko64(fd, offset, SEEK_SET)) + { + std::cerr << "(EE) ftFileMapper::ftFileMapper::writeData() Bad fseek at offset " << offset << ", fd=" << (void*)fd << ", size=" << size << ", errno=" << errno << std::endl; + return false; + } + + if (1 != fwrite(data, size, 1, fd)) + { + std::cerr << "(EE) ftFileMapper::ftFileCreator::addFileData() Bad fwrite." << std::endl; + std::cerr << "ERRNO: " << errno << std::endl; + + return false; + } + fflush(fd) ; + return true ; +} + +bool ftFileMapper::storeData(void *data, uint32_t data_size, uint64_t offset,FILE *fd) +{ + uint64_t real_offset = 0; + +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) ftFileMapper::storeData(): storing data size " << data_size << " for offset "<< offset << std::endl; +#endif + + // we compute the real place of the data in the mapped file. Several cases: + // + // 1 - the place corresponds to a mapped place + // => write there. + // 2 - the place does not correspond to a mapped place. + // 2.0 - we allocate a new chunk at the end of the file. + // 2.0.1 - the chunk corresponds to a mapped chunk somewhere before + // => we move it, and use the other chunk as writing position + // 2.0.2 - the chunk does not correspond to a mapped chunk somewhere before + // => we use it + // 2.1 - the place is in the range of existing data + // => we move the existing data at the end of the file, and update the mapping + // 2.2 - the place is outside the range of existing data + // => we allocate a new chunk at the end of the file, and write there. + // 2.2.1 - we look for the first chunk that is not already mapped before. + // + if(!computeStorageOffset(offset,real_offset)) + { + uint32_t cid = (uint32_t)(offset / (uint64_t)_chunk_size) ; + +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) real offset unknown. chunk id is " << cid << std::endl; +#endif + + uint32_t empty_chunk = allocateNewEmptyChunk(fd) ; + +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) allocated new empty chunk " << empty_chunk << std::endl; +#endif + + if(cid < _first_free_chunk && cid != empty_chunk) // the place is already occupied by some data + { +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) chunk already in use. " << std::endl; + std::cerr << "(DD) swapping with first free chunk: " << empty_chunk << std::endl; +#endif + + if(!moveChunk(cid, empty_chunk,fd)) + { + std::cerr << "(EE) ftFileMapper::writeData(): cannot move chunk " << empty_chunk << " and " << cid << std::endl ; + return false ; + } + + // Get the old chunk id that was mapping to this place + // + int oid = _data_chunks[cid] ; + + if(oid < 0) + { + std::cerr << "(EE) ftFileMapper::writeData(): cannot find chunk that was previously mapped to place " << cid << std::endl ; + return false ; + } +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) old chunk now pointing to: " << empty_chunk << std::endl; + std::cerr << "(DD) new chunk now pointing to: " << cid << std::endl; +#endif + + _mapped_chunks[cid] = cid ; // this one is in place, since we swapped it + _mapped_chunks[oid] = empty_chunk ; + _data_chunks[cid] = cid ; + _data_chunks[empty_chunk] = oid ; + } + else // allocate a new chunk at end of the file. + { +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) allocating new storage place at first free chunk: " << empty_chunk << std::endl; +#endif + + _mapped_chunks[cid] = empty_chunk ; + _data_chunks[empty_chunk] = cid ; + } + + real_offset = _mapped_chunks[cid]*_chunk_size + (offset % (uint64_t)_chunk_size) ; + } +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) real offset = " << real_offset << ", data size=" << data_size << std::endl; + std::cerr << "(DD) writing data " << std::endl; +#endif + + return writeData(real_offset,data_size,data,fd) ; +} + +uint32_t ftFileMapper::allocateNewEmptyChunk(FILE *fd_out) +{ + // look into _first_free_chunk. Is it the place of a chunk already mapped before? + // +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) ftFileMapper::getFirstEmptyChunk()" << std::endl; +#endif + + if(_mapped_chunks[_first_free_chunk] >= 0 && _mapped_chunks[_first_free_chunk] < (int)_first_free_chunk) + { + uint32_t old_chunk = _mapped_chunks[_first_free_chunk] ; + +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) first free chunk " << _first_free_chunk << " is actually mapped to " << old_chunk << ". Moving it." << std::endl; +#endif + + moveChunk(_mapped_chunks[_first_free_chunk],_first_free_chunk,fd_out) ; + _mapped_chunks[_first_free_chunk] = _first_free_chunk ; + _data_chunks[_first_free_chunk] = _first_free_chunk ; + _first_free_chunk++ ; + +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) Returning " << old_chunk << std::endl; +#endif + + return old_chunk ; + } + else + { +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) first free chunk is fine. Returning " << _first_free_chunk << ", and making room" << std::endl; +#endif + + // We need to wipe the entire chunk, since it might be moved before beign completely written, which would cause + // a fread error. + // + wipeChunk(_first_free_chunk,fd_out) ; + + return _first_free_chunk++ ; + } +} + +bool ftFileMapper::wipeChunk(uint32_t cid,FILE *fd) const +{ + uint32_t size = (cid == _mapped_chunks.size()-1)?(_file_size - cid*_chunk_size) : _chunk_size ; + + void *buf = malloc(size) ; + + if(buf == NULL) + { + std::cerr << "(EE) ftFileMapper::wipeChunk(): cannot allocate temporary buf of size " << size << std::endl; + return false ; + } + + if(fseeko64(fd, cid*_chunk_size, SEEK_SET)!= 0) + { + std::cerr << "(EE) ftFileMapper::wipeChunk(): cannot fseek file at position " << cid*_chunk_size << std::endl; + free(buf) ; + return false ; + } + + if(1 != fwrite(buf, size, 1, fd)) + { + std::cerr << "(EE) ftFileMapper::wipeChunk(): cannot write to file" << std::endl; + free(buf) ; + return false ; + } + + free(buf) ; + return true ; +} + +bool ftFileMapper::moveChunk(uint32_t to_move, uint32_t new_place,FILE *fd_out) +{ + // Read the old chunk, write at the new place + + assert(to_move != new_place) ; + + fflush(fd_out) ; +#ifdef DEBUG_FILEMAPPER + std::cerr << "(DD) ftFileMapper::moveChunk(): moving chunk " << to_move << " to place " << new_place << std::endl ; +#endif + + uint32_t new_place_size = (new_place == _mapped_chunks.size()-1)?(_file_size - (_mapped_chunks.size()-1)*_chunk_size) : _chunk_size ; + uint32_t to_move_size = (new_place == _mapped_chunks.size()-1)?(_file_size - (_mapped_chunks.size()-1)*_chunk_size) : _chunk_size ; + + uint32_t size = std::min(new_place_size,to_move_size) ; + void *buff = malloc(size) ; + + if(buff == NULL) + { + std::cerr << "(EE) ftFileMapper::moveChunk(): cannot open temporary buffer. Out of memory??" << std::endl; + return false ; + } + if(fseeko64(fd_out, to_move*_chunk_size, SEEK_SET) != 0) + { + std::cerr << "(EE) ftFileMapper::moveChunk(): cannot fseek file at position " << to_move*_chunk_size << std::endl; + return false ; + } + + size_t rd ; + if(size != (rd = fread(buff, 1, size, fd_out))) + { + std::cerr << "(EE) ftFileMapper::moveChunk(): cannot read from file" << std::endl; + std::cerr << "(EE) errno = " << errno << std::endl; + std::cerr << "(EE) feof = " << feof(fd_out) << std::endl; + std::cerr << "(EE) size = " << size << std::endl; + std::cerr << "(EE) rd = " << rd << std::endl; + return false ; + } + + if(fseeko64(fd_out, new_place*_chunk_size, SEEK_SET)!= 0) + { + std::cerr << "(EE) ftFileMapper::moveChunk(): cannot fseek file at position " << new_place*_chunk_size << std::endl; + return false ; + } + + if(1 != fwrite(buff, size, 1, fd_out)) + { + std::cerr << "(EE) ftFileMapper::moveChunk(): cannot write to file" << std::endl; + return false ; + } + free(buff) ; + + return true ; +} + +void ftFileMapper::print() const +{ + std::cerr << "ftFileMapper:: [ " ; + + for(uint32_t i=0;i<_mapped_chunks.size();++i) + { + std::cerr << _mapped_chunks[i] << " " ; + } + + std::cerr << "] - ffc = " << _first_free_chunk << " - [ "; + + for(uint32_t i=0;i<_data_chunks.size();++i) + std::cerr << _data_chunks[i] << " " ; + std::cerr << " ] " << std::endl; + +} + + diff --git a/libretroshare/src/ft/ftfilemapper.h b/libretroshare/src/ft/ftfilemapper.h new file mode 100644 index 000000000..292fa53e5 --- /dev/null +++ b/libretroshare/src/ft/ftfilemapper.h @@ -0,0 +1,98 @@ +/* + * libretroshare/src/ft/ftfilemapper.h + * + * File Transfer for RetroShare. + * + * Copyright 2011 by Cyril Soler. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "csoler@users.sourceforge.net". + * + */ + +#pragma once + +// This class implements data storage for incoming files. It provides the following functionality: +// +// - linear storage of data +// - record of which order the data is currently stored into +// - automatic (and clever) re-organisation of data as it arrives +// +// The implementation ensures that: +// - when the file is complete, the data is always ordered correctly +// - when data comes, the writes always happen within the range of existing data, plus at most one chunk. +// +// Using this class avoids writting in the middle of large files while downloading them, which removes the lag +// of RS interface when starting the DL of a large file. +// +// The re-organisation of the file data occurs seamlessly during writes, and therefore does not cause any big +// freeze at end of download. +// +// As soon as the first chunk has been downloaded, it is possible to preview a file by directly playing +// the partial file, so no change is needed to preview. +// +#include +#include +#include + +class ftFileMapper +{ + public: + // Name and size of the file to store. This will usually be a file in the Partials/ directory. + ftFileMapper(uint64_t file_size,uint32_t chunk_size) ; + + // Storage/retreive of data. All offsets are given in absolute position in the file. The class handles + // the real mapping (hence the name). + + // Stores the data in the file, at the given offset. The chunk does not necessarily exist. If not, + // the data is written at the end of the current file. If yes, it is written at the actual place. + // Returned values: + // + // true: the data has correctly been written + // false: the data could not be written + // + bool storeData(void *data, uint32_t data_size, uint64_t offset,FILE *fd) ; + + // Gets the data from the storage file. The data should be there. The data is stored into buff, which needs to be + // allocated by the client. Returned values: + // + // true: the data has correctly been read + // false: the data could not beread, or does not exist. + // + bool readData(void *buff, uint32_t data_size, uint64_t offset) ; + + // debug + void print() const ; + + private: + uint64_t _file_size ; // size of the file + uint32_t _chunk_size ; // size of chunks + uint32_t _first_free_chunk ; // first chunk in the mapped file to be available + + // List of chunk ids (0,1,2,3...) stored in the order + std::vector _mapped_chunks ; + std::vector _data_chunks ; + + bool writeData(uint64_t offset,uint32_t size,void *data,FILE *fd) const ; + bool readData(uint64_t offset,uint32_t size,void *data,FILE *fd) const ; + bool wipeChunk(uint32_t cid,FILE *fd) const ; + + bool computeStorageOffset(uint64_t offset,uint64_t& storage_offset) const ; + + bool moveChunk(uint32_t src,uint32_t dst,FILE *fd_out) ; + uint32_t allocateNewEmptyChunk(FILE *fd) ; +}; + diff --git a/libretroshare/src/tests/ft/Makefile b/libretroshare/src/tests/ft/Makefile index 628c38f78..286e69de0 100644 --- a/libretroshare/src/tests/ft/Makefile +++ b/libretroshare/src/tests/ft/Makefile @@ -8,12 +8,15 @@ DHT_TOP_DIR = ../../../../libbitdht/src include $(RS_TOP_DIR)/tests/scripts/config.mk ############################################################### -TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o fttransfermoduletest.o ftcrc32test.o ftcrossprovidercreatortest.o ftcontrollertest.o ftserver1test.o ftserver2test.o ftserver3test.o +TESTOBJ = ftfilemappertest.o ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o fttransfermoduletest.o ftcrc32test.o ftcrossprovidercreatortest.o ftcontrollertest.o ftserver1test.o ftserver2test.o ftserver3test.o -TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest fttransfermoduletest ftcrc32test ftcrossprovidercreatortest ftcontrollertest ftserver1test ftserver2test fttransfermoduletest ftserver3test +TESTS = ftfilemappertest ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest fttransfermoduletest ftcrc32test ftcrossprovidercreatortest ftcontrollertest ftserver1test ftserver2test fttransfermoduletest ftserver3test all: tests +ftfilemappertest : ftfilemappertest.o + $(CC) $(CFLAGS) -o ftfilemappertest ftfilemappertest.o $(LIBS) + ftcontrollertest : ftcontrollertest.o $(CC) $(CFLAGS) -o ftcontrollertest ftcontrollertest.o $(LIBS) diff --git a/libretroshare/src/tests/ft/ftfilemappertest.cc b/libretroshare/src/tests/ft/ftfilemappertest.cc new file mode 100644 index 000000000..69c97a3c4 --- /dev/null +++ b/libretroshare/src/tests/ft/ftfilemappertest.cc @@ -0,0 +1,91 @@ +#include "ft/ftfilemapper.h" +#include "ft/ftchunkmap.h" +#include "retroshare/rstypes.h" + +#include +#include +#include +#include +#include + +INITTEST(); + +int main() +{ + /* Use ftfilemapper to create a file with chunks downloaded on a random direction. */ + + static const std::string tmpdir = "." ; + static const std::string input_file = tmpdir+"/"+"input.bin" ; + static const std::string output_file = tmpdir+"/"+"output.bin" ; + static const uint64_t size = 1024*1024*12+234;//4357283 ; // some size. Not an integer number of chunks + static const uint32_t chunk_size = 1024*1024 ; // 1MB + + pid_t pid = getpid() ; + srand48(pid) ; + srand(pid) ; + std::cerr << "Inited random number generator with seed " << pid << std::endl; + + // 0 - create a random file in memory, of size SIZE + + void *membuf = malloc(size) ; + CHECK(membuf != NULL) ; + + for(int i=0;i