From ef9bb372b4444424e1b141c73680ac7ffd248b4a Mon Sep 17 00:00:00 2001 From: drbob Date: Wed, 22 Oct 2008 18:12:58 +0000 Subject: [PATCH] re-wrote ftfileprovider / ftfilecreator to be more simple and robust. re-wrote associated tests to be more robust. added new fttransfermoduletest.cc git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@760 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/ft/Makefile | 7 +- libretroshare/src/ft/ftcontroller.cc | 3 +- libretroshare/src/ft/ftfilecreator.cc | 749 ++++--------------- libretroshare/src/ft/ftfilecreator.h | 88 +-- libretroshare/src/ft/ftfilecreatortest.cc | 158 ++-- libretroshare/src/ft/ftfileprovider.cc | 55 +- libretroshare/src/ft/ftfileprovider.h | 9 +- libretroshare/src/ft/ftfileprovidertest.cc | 85 ++- libretroshare/src/ft/fttransfermodule.cc | 25 +- libretroshare/src/ft/fttransfermoduletest.cc | 180 +++++ 10 files changed, 573 insertions(+), 786 deletions(-) create mode 100644 libretroshare/src/ft/fttransfermoduletest.cc diff --git a/libretroshare/src/ft/Makefile b/libretroshare/src/ft/Makefile index f1de6c045..c00dd0712 100644 --- a/libretroshare/src/ft/Makefile +++ b/libretroshare/src/ft/Makefile @@ -12,10 +12,10 @@ RSOBJ = ftdata.o ftfileprovider.o ftfilecreator.o ftextralist.o \ ftcontroller.o pqitestor.o -TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o ftserver1test.o ftserver2test.o +TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o ftserver1test.o ftserver2test.o fttransfermoduletest.o -TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest ftserver1test ftserver2test +TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest ftserver1test ftserver2test fttransfermoduletest all: librs tests @@ -25,6 +25,9 @@ ftfilecreatortest : ftfilecreatortest.o ftfileprovidertest : ftfileprovidertest.o $(CC) $(CFLAGS) -o ftfileprovidertest ftfileprovidertest.o $(LIBS) +fttransfermoduletest : fttransfermoduletest.o + $(CC) $(CFLAGS) -o fttransfermoduletest fttransfermoduletest.o $(LIBS) + ftextralisttest : ftextralisttest.o $(CC) $(CFLAGS) -o ftextralisttest ftextralisttest.o $(LIBS) diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index ad407e387..c16f8af76 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -220,8 +220,7 @@ bool ftController::FileRequest(std::string fname, std::string hash, /* add in new item for download */ std::string savepath = mDownloadPath + "/" + fname; - std::string chunker = "default"; - ftFileCreator *fc = new ftFileCreator(savepath, size, hash, chunker); + ftFileCreator *fc = new ftFileCreator(savepath, size, hash, 0); ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this); /* add into maps */ diff --git a/libretroshare/src/ft/ftfilecreator.cc b/libretroshare/src/ft/ftfilecreator.cc index 5e7eccaf6..d8409919c 100644 --- a/libretroshare/src/ft/ftfilecreator.cc +++ b/libretroshare/src/ft/ftfilecreator.cc @@ -2,6 +2,9 @@ #define FILE_DEBUG 1 +#define CHUNK_MAX_AGE 30 + + /*********************************************************** * * ftFileCreator methods @@ -9,7 +12,7 @@ ***********************************************************/ ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string -hash, std::string chunker): ftFileProvider(path,size,hash) +hash, uint64_t recvd): ftFileProvider(path,size,hash) { /* * FIXME any inits to do? @@ -25,66 +28,30 @@ hash, std::string chunker): ftFileProvider(path,size,hash) std::cerr << "\thash: " << hash; std::cerr << std::endl; #endif - - initialize(chunker); + + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + + /* initialise the Transfer Lists */ + mStart = recvd; + mEnd = recvd; } -void ftFileCreator::initialize(std::string chunker) +bool ftFileCreator::getFileData(uint64_t offset, + uint32_t chunk_size, void *data) { RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - - if (chunker == "default") - fileChunker = new ftFileChunker(total_size); - else - fileChunker = new ftFileRandomizeChunker(total_size); - - fileChunker->splitFile(); + if (offset + chunk_size > mStart) + { + /* don't have the data */ + return false; + } + return ftFileProvider::getFileData(offset, chunk_size, data); } -int ftFileCreator::initializeFileAttrs() +uint64_t ftFileCreator::getRecvd() { - //RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - - /* - * check if the file exists - */ -#ifdef FILE_DEBUG - std::cerr << "ftFileCreator::initializeFileAttrs() Filename: " << file_name; - std::cerr << std::endl; -#endif - - /* - * attempt to open file in writing mode - */ - - fd = fopen(file_name.c_str(), "w+b"); - if (!fd) - { -#ifdef FILE_DEBUG - std::cerr << "ftFileCreator::initializeFileAttrs() Failed to open (w+b): " << file_name << std::endl; -#endif - return 0; - - } - - /* - * if it opened, find it's length - * move to the end - */ - if (0 != fseek(fd, 0L, SEEK_END)) - { - std::cerr << "ftFileCreator::initializeFileAttrs() Seek Failed" << std::endl; - return 0; - } - /* - * get the file length - */ - recv_size = ftell(fd); - -#ifdef FILE_DEBUG - std::cerr << "ftFileCreator::initializeFileAttrs() recv_size: " << recv_size << std::endl; -#endif - return 1; + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + return mStart; } bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data) @@ -96,28 +63,22 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data std::cerr << ", " << data << ")"; std::cerr << std::endl; #endif - RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ - /* check the status */ + /* dodgey checking outside of mutex... + * much check again inside FileAttrs(). + */ + /* Check File is open */ + if (fd == NULL) + if (!initializeFileAttrs()) + return false; - if (fd==NULL) - { -#ifdef FILE_DEBUG - std::cerr << "ftFileCreator::addFileData() initialising"; - std::cerr << std::endl; -#endif - int init = initializeFileAttrs(); - if (init ==0) { - std::cerr <<"Initialization Failed" << std::endl; - return 0; - } - } + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ /* * check its at the correct location */ - if (offset + chunk_size > this->total_size) + if (offset + chunk_size > mSize) { - chunk_size = total_size - offset; + chunk_size = mSize - offset; std::cerr <<"Chunk Size greater than total file size, adjusting chunk " << "size " << chunk_size << std::endl; } @@ -133,8 +94,6 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data long int pos; pos = ftell(fd); - std::cerr << pos << " BEFORE RECV SIZE "<< recv_size << std::endl; - /* * add the data */ @@ -144,21 +103,18 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data return 0; } - this->recv_size += chunk_size; - pos = ftell(fd); #ifdef FILE_DEBUG std::cerr << "ftFileCreator::addFileData() added Data..."; std::cerr << std::endl; - std::cerr << "recvd: " << recv_size; std::cerr << " pos: " << pos; std::cerr << std::endl; #endif /* * Notify ftFileChunker about chunks received */ - fileChunker->notifyReceived(offset,chunk_size); + notifyReceived(offset,chunk_size); /* * FIXME HANDLE COMPLETION HERE - Any better way? @@ -167,6 +123,60 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data return 1; } +int ftFileCreator::initializeFileAttrs() +{ + std::cerr << "ftFileCreator::initializeFileAttrs() Filename: "; + std::cerr << file_name; + + /* + * check if the file exists + */ + + if (ftFileProvider::initializeFileAttrs()) + { + return 1; + } + + + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + if (fd) + return 1; + + { + std::cout << + "ftFileCreator::initializeFileAttrs() Filename: " << file_name; + } + + /* + * attempt to open file + */ + + fd = fopen(file_name.c_str(), "w+b"); + if (!fd) + { + std::cout << + "ftFileCreator::initializeFileAttrs() Failed to open (w+b): "<< file_name << std::endl; + return 0; + + } + + /* + * if it opened, find it's length + * move to the end + */ + + if (0 != fseek(fd, 0L, SEEK_END)) + { + std::cerr << "ftFileCreator::initializeFileAttrs() Seek Failed" << std::endl; + return 0; + } + + uint64_t recvdsize = ftell(fd); + + std::cerr << "ftFileCreator::initializeFileAttrs() File Expected Size: " << mSize << " RecvdSize: " << recvdsize << std::endl; + + return 1; +} ftFileCreator::~ftFileCreator() { /* @@ -175,546 +185,104 @@ ftFileCreator::~ftFileCreator() } -bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk) +int ftFileCreator::notifyReceived(uint64_t offset, uint32_t chunk_size) { -#ifdef FILE_DEBUG - std::cerr << "ftFileCreator::getMissingChunk(???," << chunk << ")"; - std::cerr << std::endl; -#endif - return fileChunker->getMissingChunk(offset, chunk); -} + /* ALREADY LOCKED */ -/*********************************************************** -* -* ftFileChunker methods -* -***********************************************************/ - -ftFileChunker::ftFileChunker(uint64_t size): file_size(size), std_chunk_size(10000), monitorPeriod(30) -{ - /* - * srand for randomized version - move it to the sub-class? - */ - srand ( time(NULL) ); - aggregate_status = 0; -} - -ftFileChunker::~ftFileChunker() -{ - std::vector::iterator it; - for(unsigned int i=0; i::iterator it; + it = mChunks.find(offset); + if (it == mChunks.end()) { - uint64_t offset = index * std_chunk_size; - max_chunk_size = file_size - (index * std_chunk_size); - ftChunk *f = new ftChunk(offset,max_chunk_size,now, ftChunk::AVAIL); - allocationTable.push_back(f); + return 0; /* ignoring */ } - if (rem != 0) + ftChunk chunk = it->second; + mChunks.erase(it); + + if (chunk.chunk != chunk_size) { - ftChunk *f = new ftChunk(file_size-rem,rem,now, ftChunk::AVAIL); - allocationTable.push_back(f); - num_chunks++; + /* partial : shrink chunk */ + chunk.chunk -= chunk_size; + chunk.offset += chunk_size; + mChunks[chunk.offset] = chunk; } - /* - * DEBUGGER - * for(int j=0;jmax_chunk_size << " " << allocationTable.at(j)->chunk_status << std::endl; - * } - */ + /* if we've cleaned up chunks, must update counters */ + it = mChunks.find(chunk.offset); + if (it == mChunks.end()) + { + mStart = mEnd; + } + else if (it == mChunks.begin()) + { + mStart += it->second.offset; + } + + return 1; } -/* - * This method sets the offset, chunk may be reset if needed - */ +bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk) +{ + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + std::cerr << "ffc::getMissingChunk(...,"<< chunk << ")"<< std::endl; -bool ftFileChunker::getMissingChunk(uint64_t &offset, uint32_t &chunk) -{ - RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/ + /* check start point */ - unsigned int i =0; - bool found = false; - int chunks_after = 0; - int chunks_rem = 0; + if (mStart == mSize) + { + std::cerr << "ffc::getMissingChunk() File Done"; + std::cerr << std::endl; + return false; + } -#ifdef FILE_DEBUG - std::cerr << "ftFileChunker::getMissingChunk(???," << chunk << ")"; + /* check for freed chunks */ + time_t ts = time(NULL); + time_t old = ts-CHUNK_MAX_AGE; + + std::map::iterator it; + for(it = mChunks.begin(); it != mChunks.end(); it++) + { + /* very simple algorithm */ + if (it->second.ts < old) + { + std::cerr << "ffc::getMissingChunk() ReAlloc"; + std::cerr << std::endl; + + /* retry this one */ + it->second.ts = ts; + chunk = it->second.chunk; + offset = it->second.offset; + + return true; + } + } + + std::cerr << "ffc::getMissingChunk() new Alloc"; + std::cerr << " mStart: " << mStart << " mEnd: " << mEnd; + std::cerr << "mSize: " << mSize; std::cerr << std::endl; -#endif - /* - * This signals file completion - * FIXME Does it need to be more explicit - */ - - if(aggregate_status == num_chunks * ftChunk::RECEIVED) - { -#ifdef FILE_DEBUG - std::cerr << "completed ??"; - std::cerr << std::endl; -#endif - return found; - } + /* else allocate a new chunk */ + if (mSize - mEnd < chunk) + chunk = mSize - mEnd; + offset = mEnd; + mEnd += chunk; - while(imax_chunk_size >=chunk) - { - -#ifdef FILE_DEBUG - std::cerr << "ftFileChunker pre alloc(" << i << ")"; - std::cerr << std::endl; - std::cerr << "\toffset: " << allocationTable.at(i)->offset; - std::cerr << std::endl; - std::cerr << "\tmax_chunk: " << allocationTable.at(i)->max_chunk_size; - std::cerr << std::endl; - std::cerr << "\treq_chunk: " << chunk; - std::cerr << std::endl; -#endif - - - offset = allocationTable.at(i)->offset; - chunks_after = chunk/std_chunk_size; //10KB - - /* - * FIXME Handling remaining chunk < 10KB - */ - - //if (chunk < - chunks_rem = chunk % std_chunk_size; - chunk -= chunks_rem; - /*std::cout << "Found " << chunk << " at " << i << " "<< chunks_after << std::endl;*/ - allocationTable.at(i)->max_chunk_size=0; - allocationTable.at(i)->timestamp = time(NULL); - allocationTable.at(i)->chunk_status = ftChunk::ALLOCATED; - - -#ifdef FILE_DEBUG - std::cerr << "ftFileChunker postalloc(" << i << ")"; - std::cerr << std::endl; - std::cerr << "\tchunks_after: " << chunks_after; - std::cerr << std::endl; - std::cerr << "\tchunks_rem: " << chunks_after; - std::cerr << std::endl; - std::cerr << "\tchunk: " << chunks_after; - std::cerr << std::endl; -#endif - - found = true; - break; - } - i++; - } - - /* if we get here, there is no available chunk bigger - * than requested ... - * NB: Request size should be a larger than std_chunk_size. - * So Edge (sub chunk allocation) condition is handled here. - * - * Find largest remaining chunk. - */ - - if (!found) - { -#ifdef FILE_DEBUG - std::cerr << "ftFileChunker::getMissingChuck() FULL CHUNK not found"; - std::cerr << std::endl; -#endif - i=0; - uint64_t max = allocationTable.at(i)->max_chunk_size; - uint64_t size = max; - int maxi = -1; - if (max > 0) - { - maxi = 0; - } - - while(imax_chunk_size; - std::cerr << std::endl; -#endif - size = allocationTable.at(i)->max_chunk_size; - if(size > max) - { - max = allocationTable.at(i)->max_chunk_size; - maxi = i; - } - i++; - } - if (maxi > -1) //maxi or max - { - -#ifdef FILE_DEBUG - std::cerr << "Biggest Avail Chunk: " << max; - std::cerr << std::endl; -#endif - - offset = allocationTable.at(maxi)->offset; - chunk = allocationTable.at(maxi)->max_chunk_size; - chunks_after = chunk/std_chunk_size; //10KB - - /* Handle end condition ... - * max_chunk_size < std_chunk_size - * Trim if not end condition. - */ - if (chunks_after > 0) - { - chunks_rem = chunk % std_chunk_size; - chunk -= chunks_rem; - } - else - { - /* end condition */ - chunks_after = 1; - } - - allocationTable.at(maxi)->max_chunk_size=0; - allocationTable.at(maxi)->timestamp = time(NULL); - allocationTable.at(maxi)->chunk_status = ftChunk::ALLOCATED; - found = true; - i = maxi; - } - - } //if not found - - - if (found) - { - // i represents index to chunk(s) we will use. - // chunks_after is the count of how many we will use. - - std::cout << "Chunks remaining " << chunks_rem << std::endl; - /* - * update all previous chunks max available size - * Expensive? Can it be smarter FIXME - */ - - /* drbob: Think this is wrong? - * disabling... - * - for(unsigned int j=0;jmax_chunk_size >0) - allocationTable.at(j)->max_chunk_size -= chunk; - } - * - */ - - - for(unsigned int j=i;jmax_chunk_size = 0; - allocationTable.at(j)->chunk_status = ftChunk::ALLOCATED; - } - - // DEBUGGER - Uncomment - for(unsigned int j=0;jmax_chunk_size << " " << allocationTable.at(j)->chunk_status << std::endl; - } - - } - else - { -#ifdef FILE_DEBUG - std::cerr << "ftFileChunker::getMissingChuck() not found"; - std::cerr << std::endl; -#endif - } - - - return found; + mChunks[offset] = ftChunk(offset, chunk, ts); + return true; } -/* - * This should run on a separate thread when ftFileChunker is initialized - * FIXME Implemet DrBob's suggestion of request-fired check instead of dedicated - * thread - */ -int ftFileChunker::monitor() -{ - int reset = 0; - uint32_t prev_size = 0; - uint32_t size = 0; - - std::cout<<"Running monitor.."<= 0;) - { - if((allocationTable.at(j)->chunk_status == ftChunk::ALLOCATED) && - (allocationTable.at(j)->timestamp - time(NULL) > 30)) - { - allocationTable.at(j)->chunk_status = ftChunk::AVAIL; - if (j == allocationTable.size()-1) - { - /* at end */ - prev_size = 0; - size = file_size % std_chunk_size; - if (size == 0) - { - size = std_chunk_size; - } - } - else - { - prev_size = allocationTable.at(j+1)->max_chunk_size; - size = std_chunk_size; - } - - allocationTable.at(j)->max_chunk_size = size + prev_size; - prev_size = allocationTable.at(j)->max_chunk_size; - - for(j--; j >= 0; j--) - { - if (allocationTable.at(j)->chunk_status != ftChunk::AVAIL) - break; - - allocationTable.at(j)->max_chunk_size += prev_size; - prev_size = allocationTable.at(j)->max_chunk_size; - reset++; - } - } - else - { - j--; - } - } - return reset; -} - -void ftFileChunker::setMonitorPeriod(int period) -{ - monitorPeriod = period; -} - -void ftFileChunker::run() -{ - - while(1) - { - - for(int i = 0; i < monitorPeriod; i++) - { - -/******************** WINDOWS/UNIX SPECIFIC PART ******************/ -#ifndef WINDOWS_SYS - sleep(1); -#else - - Sleep(1000); -#endif -/******************* WINDOWS/UNIX SPECIFIC PART ******************/ - } - monitor(); - } - -} - -int ftFileChunker::notifyReceived(uint64_t offset, uint32_t chunk_size) -{ - RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/ - int index = offset / std_chunk_size; - - /* should receive a multiple of chunk_size.... */ - uint32_t chunks = chunk_size / std_chunk_size; - uint32_t rem_chunk = chunk_size % std_chunk_size; - -#ifdef FILE_DEBUG - std::cerr << "ftFileChunkerr::notifyReceived("; - std::cerr << offset; - std::cerr << ", " << chunk_size << ")"; - std::cerr << std::endl; - std::cerr << "\t# complete chunks: " << chunks; - std::cerr << std::endl; - std::cerr << "\trem_chunk: " << rem_chunk; - std::cerr << std::endl; -#endif - - if(allocationTable.at(index)->chunk_status == ftChunk::ALLOCATED){ - allocationTable.at(index)->chunk_status = ftChunk::RECEIVED; - aggregate_status += ftChunk::RECEIVED; - -#ifdef FILE_DEBUG - std::cerr << "ftFileChunkerr::notifyReceived()"; - std::cerr << " flagged as RECVD"; - std::cerr << std::endl; -#endif - - } - return aggregate_status; -} - -/*********************************************************** -* -* ftFileRandomizeChunker methods -* -***********************************************************/ - -ftFileRandomizeChunker::ftFileRandomizeChunker(uint64_t size): -ftFileChunker(size) -{ - -} - -ftFileRandomizeChunker::~ftFileRandomizeChunker() -{ - -} - -bool ftFileRandomizeChunker::getMissingChunk(uint64_t &offset, uint32_t &chunk) { - - RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/ - std::cerr << "Calling getMissingChunk with chunk="<< chunk << std::endl; - unsigned int i =0; - bool found = false; - int chunks_after = 0; - int chunks_rem = 0; - - if(aggregate_status == num_chunks * ftChunk::RECEIVED) - return found; - - std::vector randomIndex; - while(imax_chunk_size >=chunk){ - randomIndex.push_back(i); - } - i++; - } - - /* - * FIXME test sufficiently to make sure its picking every index - */ - if (randomIndex.size()>0) - { - int rnum = rand() % randomIndex.size(); - i = randomIndex.at(rnum); - std::cout << "i=" <offset; - chunks_after = chunk/std_chunk_size; //10KB - chunks_rem = chunk % std_chunk_size; - chunk -= chunks_rem; - std::cout << "Found " << chunk << " at index =" << i << " "<< chunks_after << std::endl; - allocationTable.at(i)->max_chunk_size=0; - allocationTable.at(i)->timestamp = time(NULL); - allocationTable.at(i)->chunk_status = ftChunk::ALLOCATED; - found = true; - } - - if (!found) - { - i=0; - uint64_t min = allocationTable.at(i)->max_chunk_size - chunk; - uint64_t diff = min; - int mini = -1; - while(imax_chunk_size-chunk; - if(diff <= min && diff >0) - { - min = allocationTable.at(i)->max_chunk_size - chunk; - mini = i; - } - i++; - } - if (mini > -1) - { - offset = allocationTable.at(mini)->offset; - chunk = allocationTable.at(mini)->max_chunk_size; - chunks_after = chunk/std_chunk_size; //10KB - chunks_rem = chunk % std_chunk_size; - chunk -= chunks_rem; - allocationTable.at(mini)->max_chunk_size=0; - allocationTable.at(mini)->timestamp = time(NULL); - allocationTable.at(mini)->chunk_status = ftChunk::ALLOCATED; - found = true; - } - - } //if not found - - if (found) - { - // update all previous chunks max available size - for(unsigned int j=0;jmax_chunk_size >0) - allocationTable.at(j)->max_chunk_size -= chunk; - } - - for(unsigned int j=i;jmax_chunk_size = 0; - allocationTable.at(j)->chunk_status = ftChunk::ALLOCATED; - } - - /* for(int j=0;jmax_chunk_size << " " << allocationTable.at(j)->chunk_status << std::endl; - }*/ - } - return found; -} /*********************************************************** * * ftChunk methods * ***********************************************************/ -ftChunk::ftChunk(uint64_t offset,uint64_t chunk_size,time_t time, ftChunk::Status s) : offset(offset), max_chunk_size(chunk_size), timestamp(time), chunk_status(s) +ftChunk::ftChunk(uint64_t ioffset,uint64_t size,time_t now) + : offset(ioffset), chunk(size), ts(now) { } @@ -723,3 +291,4 @@ ftChunk::~ftChunk() { } + diff --git a/libretroshare/src/ft/ftfilecreator.h b/libretroshare/src/ft/ftfilecreator.h index a0733bd85..d3874746c 100644 --- a/libretroshare/src/ft/ftfilecreator.h +++ b/libretroshare/src/ft/ftfilecreator.h @@ -33,100 +33,56 @@ * */ #include "ftfileprovider.h" -#include "util/rsthreads.h" -#include +#include class ftChunk; -class ftFileChunker; class ftFileCreator: public ftFileProvider { public: - ftFileCreator(std::string savepath, uint64_t size, std::string hash,std::string chunker); + ftFileCreator(std::string savepath, uint64_t size, std::string hash, uint64_t recvd); ~ftFileCreator(); - /* - * overloaded from FileProvider FIXME - * virtual bool getFileData(uint64_t offset, uint32_t chunk_size, void *data); - * - */ + /* overloaded from FileProvider */ +virtual bool getFileData(uint64_t offset, uint32_t chunk_size, void *data); + uint64_t getRecvd(); - /* - * FIXME: initializeFileAttrs - * Should this be a over-ridden version of ftfile provider? - * What happens in the case of simultaneous upload and download? - */ - int initializeFileAttrs(); /* * creation functions for FileCreator */ + bool getMissingChunk(uint64_t &offset, uint32_t &chunk); bool addFileData(uint64_t offset, uint32_t chunk_size, void *data); +protected: + +virtual int initializeFileAttrs(); + private: + + int notifyReceived(uint64_t offset, uint32_t chunk_size); /* * structure to track missing chunks */ - uint64_t recv_size; - std::string hash; - ftFileChunker *fileChunker; - void initialize(std::string); - RsMutex ftcMutex; -}; + uint64_t mStart; + uint64_t mEnd; -/* - This class can either be specialized to follow a different splitting - policy or have an argument to indicate different policies -*/ -class ftFileChunker : public RsThread { -public: - /* - * FIXME Does filechunker require hash?? - */ - ftFileChunker(uint64_t size); - virtual ~ftFileChunker(); - - /* - * Breaks up the file into evenly sized chunks - * Initializes all chunks to never_requested - */ - int splitFile(); - virtual void run(); - virtual bool getMissingChunk(uint64_t &offset, uint32_t &chunk); - bool getMissingChunkRandom(uint64_t &offset, uint32_t &chunk); - int notifyReceived(uint64_t offset, uint32_t chunk_size); - int monitor(); - void setMonitorPeriod(int period); -protected: - uint64_t file_size; - uint64_t num_chunks; - uint64_t std_chunk_size; - std::vector allocationTable; - unsigned int aggregate_status; - int monitorPeriod; - RsMutex chunkerMutex; /********** STACK LOCKED MTX ******/ -}; - -class ftFileRandomizeChunker : public ftFileChunker { -public: - ftFileRandomizeChunker(uint64_t size); - virtual bool getMissingChunk(uint64_t &offset, uint32_t &chunk); - ~ftFileRandomizeChunker(); + std::map mChunks; }; class ftChunk { public: - enum Status {AVAIL, ALLOCATED, RECEIVED}; - ftChunk(uint64_t,uint64_t,time_t,Status); - uint64_t offset; - uint64_t max_chunk_size; - time_t timestamp; - Status chunk_status; + ftChunk(uint64_t,uint64_t,time_t); + ftChunk():offset(0), chunk(0), ts(0) {} ~ftChunk(); + + uint64_t offset; + uint64_t chunk; + time_t ts; }; -#endif // FT_FILE_PROVIDER_HEADER +#endif // FT_FILE_CREATOR_HEADER diff --git a/libretroshare/src/ft/ftfilecreatortest.cc b/libretroshare/src/ft/ftfilecreatortest.cc index ada2bd8be..3fb770e8a 100644 --- a/libretroshare/src/ft/ftfilecreatortest.cc +++ b/libretroshare/src/ft/ftfilecreatortest.cc @@ -1,87 +1,89 @@ #include "ftfilecreator.h" -main(){ - uint64_t offset; - uint32_t csize; - /*Test file creator*/ - ftFileCreator fcreator("somefile",100000,"hash","default"); - csize = 40000; - if (fcreator.getMissingChunk(offset, csize)){ - std::cout << "Missing Chunk's offset=" << offset << " chunk_size=" << csize << std::endl; - } - - csize = 10000; - if (fcreator.getMissingChunk(offset, csize)){ - std::cout << "Missing Chunk's offset=" << offset << " chunk_size=" << csize << std::endl; - } - - csize = 15000; - if (fcreator.getMissingChunk(offset, csize)){ - std::cout << "Missing Chunk's offset=" << offset << " chunk_size=" << csize << std::endl; - } - - - std::cout << "Test file creator adding data to file out-of-order\n"; - char* alpha = "abcdefghij"; - std::cerr << "Call to addFileData =" << fcreator.addFileData(10,10,alpha ); - char* numerical = "1234567890"; - std::cerr << "Call to addFileData =" << fcreator.addFileData(0,10,numerical ); - +#include "util/utest.h" - /* Test file creator can write out of order/randomized chunker */ - ftFileCreator franc("somefile1",50000,"hash", "randomize"); - csize = 30000; - if (franc.getMissingChunk(offset, csize)){ - std::cout << "Offset " << offset << " Csize " << csize << std::endl; - } +INITTEST(); - char* allA = (char*) malloc(csize); - for(int i=0;igetMissingChunk(offset, chunk); + std::cerr << "Allocated Offset: " << offset << " chunk: " << chunk << std::endl; + + CHECK(offset < max_offset); + sleep(1); + } + + std::cerr << "Expect Repeats now"; + std::cerr << std::endl; + + for(i = 0; i < max_timeout; i++) + { + creator->getMissingChunk(offset, chunk); + std::cerr << "Allocated Offset: " << offset << " chunk: " << chunk << std::endl; + + CHECK(offset < max_offset); + sleep(1); + } + REPORT("Chunk Queue"); + + return 1; +} + + +int test_fill(ftFileCreator *creator) +{ + uint32_t chunk = 1000; + uint64_t offset = 0; + + uint64_t init_size = creator->getFileSize(); + uint64_t init_trans = creator->getRecvd(); + std::cerr << "Initial FileSize: " << init_size << std::endl; + std::cerr << "Initial Transferred:" << init_trans << std::endl; + + while(creator->getMissingChunk(offset, chunk)) + { + /* give it too them */ + void *data = malloc(chunk); + creator->addFileData(offset, chunk, data); + free(data); + } + + uint64_t end_size = creator->getFileSize(); + uint64_t end_trans = creator->getRecvd(); + std::cerr << "End FileSize: " << end_size << std::endl; + std::cerr << "End Transferred:" << end_trans << std::endl; + CHECK(init_size == end_size); + CHECK(end_trans == end_size); + + REPORT("Test Fill"); + + return 1; +} + diff --git a/libretroshare/src/ft/ftfileprovider.cc b/libretroshare/src/ft/ftfileprovider.cc index 1321d972a..5949c9d0e 100644 --- a/libretroshare/src/ft/ftfileprovider.cc +++ b/libretroshare/src/ft/ftfileprovider.cc @@ -3,8 +3,8 @@ #include "util/rsdir.h" ftFileProvider::ftFileProvider(std::string path, uint64_t size, std::string -hash) : total_size(size), hash(hash), file_name(path), fd(NULL) { - +hash) : mSize(size), hash(hash), file_name(path), fd(NULL) +{ } ftFileProvider::~ftFileProvider(){ @@ -13,20 +13,29 @@ ftFileProvider::~ftFileProvider(){ } } +bool ftFileProvider::fileOk() +{ + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + return (fd != NULL); +} + std::string ftFileProvider::getHash() { + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ return hash; } uint64_t ftFileProvider::getFileSize() { - return total_size; + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + return mSize; } bool ftFileProvider::FileDetails(FileInfo &info) { + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ info.hash = hash; - info.size = total_size; + info.size = mSize; info.path = file_name; info.fname = RsDirUtil::getTopDir(file_name); @@ -38,17 +47,15 @@ bool ftFileProvider::FileDetails(FileInfo &info) bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *data) { - RsStackMutex stack(ftPMutex); /********** STACK LOCKED MTX ******/ - if (fd==NULL) - { - int init = initializeFileAttrs(); - if (init ==0) - { - std::cerr <<"Initialization Failed" << std::endl; - return 0; - } - } - + /* dodgey checking outside of mutex... + * much check again inside FileAttrs(). + */ + if (fd == NULL) + if (!initializeFileAttrs()) + return false; + + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + /* * Use private data, which has a pointer to file, total size etc */ @@ -59,9 +66,9 @@ bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *dat int data_size = chunk_size; long base_loc = offset; - if (base_loc + data_size > total_size) + if (base_loc + data_size > mSize) { - data_size = total_size - base_loc; + data_size = mSize - base_loc; std::cerr <<"Chunk Size greater than total file size, adjusting chunk size " << data_size << std::endl; } @@ -108,6 +115,14 @@ bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *dat int ftFileProvider::initializeFileAttrs() { + std::cerr << "ftFileProvider::initializeFileAttrs() Filename: "; + std::cerr << file_name; + + + RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/ + if (fd) + return 1; + /* * check if the file exists */ @@ -130,7 +145,6 @@ int ftFileProvider::initializeFileAttrs() } - /* * if it opened, find it's length * move to the end @@ -142,6 +156,9 @@ int ftFileProvider::initializeFileAttrs() return 0; } - total_size = ftell(fd); + uint64_t recvdsize = ftell(fd); + + std::cerr << "ftFileProvider::initializeFileAttrs() File Expected Size: " << mSize << " RecvdSize: " << recvdsize << std::endl; + return 1; } diff --git a/libretroshare/src/ft/ftfileprovider.h b/libretroshare/src/ft/ftfileprovider.h index 80da1b8f5..0d446a21e 100644 --- a/libretroshare/src/ft/ftfileprovider.h +++ b/libretroshare/src/ft/ftfileprovider.h @@ -40,15 +40,18 @@ class ftFileProvider public: ftFileProvider(std::string path, uint64_t size, std::string hash); virtual ~ftFileProvider(); + virtual bool getFileData(uint64_t offset, uint32_t chunk_size, void *data); virtual bool FileDetails(FileInfo &info); std::string getHash(); uint64_t getFileSize(); + bool fileOk(); protected: - virtual int initializeFileAttrs(); - uint64_t total_size; +virtual int initializeFileAttrs(); /* does for both */ + + uint64_t mSize; std::string hash; std::string file_name; FILE *fd; @@ -65,7 +68,7 @@ protected: /* * Mutex Required for stuff below */ - RsMutex ftPMutex; + RsMutex ftcMutex; }; diff --git a/libretroshare/src/ft/ftfileprovidertest.cc b/libretroshare/src/ft/ftfileprovidertest.cc index de26d21b9..d67cb4764 100644 --- a/libretroshare/src/ft/ftfileprovidertest.cc +++ b/libretroshare/src/ft/ftfileprovidertest.cc @@ -1,33 +1,72 @@ #include "ftfileprovider.h" +#include "ftfilecreator.h" -int main(){ - ftFileProvider fp("dummy.txt",1,"ABCDEF"); - char data[2]; - long offset = 0; - for (int i=0;i<10;i++) { - - if (fp.getFileData(offset,2,&data)){ - std::cout <<"Recv data " << data[0] << std::endl; +#include "util/utest.h" + + +INITTEST() + +int main() +{ + + /* create a random file */ + uint64_t size = 100000; + uint32_t max_chunk = 10000; + uint32_t chunk = 1000; + uint64_t offset = 0; + + std::string filename = "/tmp/ft_test.dta"; + std::string filename2 = "/tmp/ft_test.dta.dup"; + + /* use creator to make it */ + + void *data = malloc(max_chunk); + + ftFileCreator *creator = new ftFileCreator(filename, size, "hash", 0); + for(offset = 0; offset != size; offset += chunk) + { + if (!creator->addFileData(offset, chunk, data)) + { + FAILED("Create Test Data File"); + std::cerr << "Failed to add data (CREATE)"; + std::cerr << std::endl; } - else { - std::cout <<"Recv no data." << std::endl; - } - offset+=2; } + delete creator; - - ftFileProvider fp1("dummy1.txt",3,"ABCDEF"); - char data1[3]; - offset = 0; - for (int i=0;i<10;i++) { - - if (fp1.getFileData(offset,2,&data1)){ - std::cout <<"Recv data " << data1[0] << std::endl; + std::cerr << "Created file: " << filename << " of size: " << size; + std::cerr << std::endl; + + /* load it with file provider */ + creator = new ftFileCreator(filename2, size, "hash", 0); + ftFileProvider *provider = new ftFileProvider(filename, size, "hash"); + + /* create duplicate with file creator */ + + while(creator->getMissingChunk(offset, chunk)) + { + if (!provider->getFileData(offset, chunk, data)) + { + FAILED("Read from Test Data File"); + std::cerr << "Failed to get data"; + std::cerr << std::endl; } - else { - std::cout <<"Revc no data" << std::endl; + + if (!creator->addFileData(offset, chunk, data)) + { + FAILED("Write to Duplicate"); + std::cerr << "Failed to add data"; + std::cerr << std::endl; } - offset+=2; + + std::cerr << "Transferred: " << chunk << " @ " << offset; + std::cerr << std::endl; + + + /* reset chunk size */ + chunk = (uint64_t) max_chunk * (rand() / (1.0 + RAND_MAX)); + + std::cerr << "ChunkSize = " << chunk << std::endl; } return 1; } diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index 26f4b057c..cca2d4fcd 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -140,16 +140,35 @@ bool ftTransferModule::recvFileData(std::string peerId, uint64_t offset, mit = mFileSources.find(peerId); if (mit == mFileSources.end()) - return false; + { +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::recvFileData()"; + std::cerr << " peer not found in sources"; + std::cerr << std::endl; +#endif + return false; + } if ((mit->second).state != PQIPEER_DOWNLOADING) - return false; + { +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::recvFileData()"; + std::cerr << " peer not downloading???"; + std::cerr << std::endl; +#endif + //return false; + } if (offset != ((mit->second).offset + (mit->second).receivedSize)) { //fix me //received data not expected - return false; +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::recvFileData()"; + std::cerr << " offset != offset + recvdSize"; + std::cerr << std::endl; +#endif + return false; } (mit->second).receivedSize += chunk_size; diff --git a/libretroshare/src/ft/fttransfermoduletest.cc b/libretroshare/src/ft/fttransfermoduletest.cc new file mode 100644 index 000000000..0370a841e --- /dev/null +++ b/libretroshare/src/ft/fttransfermoduletest.cc @@ -0,0 +1,180 @@ +/* + * libretroshare/src/ft: fttransfermoduletest.cc + * + * File Transfer for RetroShare. + * + * Copyright 2008 by Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +/* + * Test for Multiplexor..... + * As this is a key middle component, it is hard to test without other bits. + * It relies on ftFileProvider/ftFileCreator/ftTransferModule... + * + * And has dummy ftDataSend and ftSearch. + * + */ + +#ifdef WIN32 +#include "util/rswin.h" +#endif + +#include "ft/ftextralist.h" +#include "ft/ftdatamultiplex.h" +#include "ft/ftfilesearch.h" + +#include "ftfileprovider.h" +#include "ftfilecreator.h" +#include "ftcontroller.h" +#include "fttransfermodule.h" + +#include "util/utest.h" + +INITTEST() + + +void do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list &files); + + +void usage(char *name) +{ + std::cerr << "Usage: " << name << " [-p ] [-d ] [ ... ] "; + std::cerr << std::endl; +} + +int main(int argc, char **argv) +{ + int c; + uint32_t period = 1; + uint32_t dPeriod = 600; /* default 10 minutes */ + + std::list fileList; + + while(-1 != (c = getopt(argc, argv, "d:p:"))) + { + switch (c) + { + case 'p': + period = atoi(optarg); + break; + case 'd': + dPeriod = atoi(optarg); + break; + default: + usage(argv[0]); + break; + } + } + + if (optind >= argc) + { + std::cerr << "Missing Files" << std::endl; + usage(argv[0]); + } + + for(; optind < argc; optind++) + { + std::cerr << "Adding: " << argv[optind] << std::endl; + fileList.push_back(std::string(argv[optind])); + } + + ftExtraList *eList = new ftExtraList(); + eList->start(); + + ftSearch *ftsd = new ftSearchDummy(); + ftFileSearch *ftfs = new ftFileSearch(); + + ftfs-> addSearchMode(ftsd, RS_FILE_HINTS_LOCAL); + ftfs-> addSearchMode(eList, RS_FILE_HINTS_EXTRA); + + ftDataSendPair *ftds1 = new ftDataSendPair(NULL); + ftDataSendPair *ftds2 = new ftDataSendPair(NULL); + + /* setup Actual Test bit */ + ftDataMultiplex *ftmplex1 = new ftDataMultiplex("ownId", ftds2, ftfs); + ftDataMultiplex *ftmplex2 = new ftDataMultiplex("ownId", ftds1, ftfs); + + ftds1->mDataRecv = ftmplex1; + ftds2->mDataRecv = ftmplex2; + + ftmplex1->start(); + ftmplex2->start(); + + /* Setup Search with some valid results */ + /* Request Data */ + + /* now work the thread */ + std::list::iterator it; + uint32_t flags = 0; + for(it = fileList.begin(); it != fileList.end(); it++) + { + eList->hashExtraFile(*it, dPeriod, flags); + } + + if (fileList.size() < 1) + { + std::cerr << "come on, give us some files..."; + std::cerr << std::endl; + return 0; + } + + /* now request files from ftDataMultiplex + * by adding in a ftTransferModule! + */ + std::string filename = *(fileList.begin()); + + /* wait for file to hash */ + FileInfo info; + while(!eList->hashExtraFileDone(filename, info)) + { + std::cerr << "Waiting for file to hash"; + std::cerr << std::endl; + sleep(1); + } + + std::string savename = "/tmp/" + info.fname; + ftFileCreator *creator = new ftFileCreator(savename, info.size, info.hash, 0); + ftController *controller = NULL; + + ftTransferModule *transfer = new ftTransferModule(creator, ftmplex1, controller); + + ftmplex1->addTransferModule(transfer, creator); + + std::list peerIds; + peerIds.push_back("ownId2"); + + transfer->setFileSources(peerIds); + transfer->setPeerState("ownId2", PQIPEER_IDLE, 1000); + //transfer->resumeTransfer(); + + /* check file progress */ + while(1) + { + std::cerr << "File Transfer Status"; + std::cerr << std::endl; + + std::cerr << "Transfered: " << creator->getRecvd(); + std::cerr << std::endl; + + transfer->tick(); + sleep(1); + } +} +