mirror of
https://github.com/RetroShare/RetroShare.git
synced 2024-10-01 02:35:48 -04:00
re-wrote ftfileprovider / ftfilecreator to be more simple and robust.
re-wrote associated tests to be more robust. added new fttransfermoduletest.cc git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@760 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
drbob
parent
26661ffb1c
commit
ef9bb372b4
@ -12,10 +12,10 @@ RSOBJ = ftdata.o ftfileprovider.o ftfilecreator.o ftextralist.o \
|
||||
ftcontroller.o pqitestor.o
|
||||
|
||||
|
||||
TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o ftserver1test.o ftserver2test.o
|
||||
TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o ftserver1test.o ftserver2test.o fttransfermoduletest.o
|
||||
|
||||
|
||||
TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest ftserver1test ftserver2test
|
||||
TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest ftserver1test ftserver2test fttransfermoduletest
|
||||
|
||||
all: librs tests
|
||||
|
||||
@ -25,6 +25,9 @@ ftfilecreatortest : ftfilecreatortest.o
|
||||
ftfileprovidertest : ftfileprovidertest.o
|
||||
$(CC) $(CFLAGS) -o ftfileprovidertest ftfileprovidertest.o $(LIBS)
|
||||
|
||||
fttransfermoduletest : fttransfermoduletest.o
|
||||
$(CC) $(CFLAGS) -o fttransfermoduletest fttransfermoduletest.o $(LIBS)
|
||||
|
||||
ftextralisttest : ftextralisttest.o
|
||||
$(CC) $(CFLAGS) -o ftextralisttest ftextralisttest.o $(LIBS)
|
||||
|
||||
|
||||
@ -220,8 +220,7 @@ bool ftController::FileRequest(std::string fname, std::string hash,
|
||||
|
||||
/* add in new item for download */
|
||||
std::string savepath = mDownloadPath + "/" + fname;
|
||||
std::string chunker = "default";
|
||||
ftFileCreator *fc = new ftFileCreator(savepath, size, hash, chunker);
|
||||
ftFileCreator *fc = new ftFileCreator(savepath, size, hash, 0);
|
||||
ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this);
|
||||
|
||||
/* add into maps */
|
||||
|
||||
@ -2,6 +2,9 @@
|
||||
|
||||
#define FILE_DEBUG 1
|
||||
|
||||
#define CHUNK_MAX_AGE 30
|
||||
|
||||
|
||||
/***********************************************************
|
||||
*
|
||||
* ftFileCreator methods
|
||||
@ -9,7 +12,7 @@
|
||||
***********************************************************/
|
||||
|
||||
ftFileCreator::ftFileCreator(std::string path, uint64_t size, std::string
|
||||
hash, std::string chunker): ftFileProvider(path,size,hash)
|
||||
hash, uint64_t recvd): ftFileProvider(path,size,hash)
|
||||
{
|
||||
/*
|
||||
* FIXME any inits to do?
|
||||
@ -25,66 +28,30 @@ hash, std::string chunker): ftFileProvider(path,size,hash)
|
||||
std::cerr << "\thash: " << hash;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
initialize(chunker);
|
||||
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
/* initialise the Transfer Lists */
|
||||
mStart = recvd;
|
||||
mEnd = recvd;
|
||||
}
|
||||
|
||||
void ftFileCreator::initialize(std::string chunker)
|
||||
bool ftFileCreator::getFileData(uint64_t offset,
|
||||
uint32_t chunk_size, void *data)
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
if (chunker == "default")
|
||||
fileChunker = new ftFileChunker(total_size);
|
||||
else
|
||||
fileChunker = new ftFileRandomizeChunker(total_size);
|
||||
|
||||
fileChunker->splitFile();
|
||||
if (offset + chunk_size > mStart)
|
||||
{
|
||||
/* don't have the data */
|
||||
return false;
|
||||
}
|
||||
return ftFileProvider::getFileData(offset, chunk_size, data);
|
||||
}
|
||||
|
||||
int ftFileCreator::initializeFileAttrs()
|
||||
uint64_t ftFileCreator::getRecvd()
|
||||
{
|
||||
//RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
/*
|
||||
* check if the file exists
|
||||
*/
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileCreator::initializeFileAttrs() Filename: " << file_name;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* attempt to open file in writing mode
|
||||
*/
|
||||
|
||||
fd = fopen(file_name.c_str(), "w+b");
|
||||
if (!fd)
|
||||
{
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileCreator::initializeFileAttrs() Failed to open (w+b): " << file_name << std::endl;
|
||||
#endif
|
||||
return 0;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* if it opened, find it's length
|
||||
* move to the end
|
||||
*/
|
||||
if (0 != fseek(fd, 0L, SEEK_END))
|
||||
{
|
||||
std::cerr << "ftFileCreator::initializeFileAttrs() Seek Failed" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
/*
|
||||
* get the file length
|
||||
*/
|
||||
recv_size = ftell(fd);
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileCreator::initializeFileAttrs() recv_size: " << recv_size << std::endl;
|
||||
#endif
|
||||
return 1;
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
return mStart;
|
||||
}
|
||||
|
||||
bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data)
|
||||
@ -96,28 +63,22 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data
|
||||
std::cerr << ", " << data << ")";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
/* check the status */
|
||||
/* dodgey checking outside of mutex...
|
||||
* much check again inside FileAttrs().
|
||||
*/
|
||||
/* Check File is open */
|
||||
if (fd == NULL)
|
||||
if (!initializeFileAttrs())
|
||||
return false;
|
||||
|
||||
if (fd==NULL)
|
||||
{
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileCreator::addFileData() initialising";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
int init = initializeFileAttrs();
|
||||
if (init ==0) {
|
||||
std::cerr <<"Initialization Failed" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
/*
|
||||
* check its at the correct location
|
||||
*/
|
||||
if (offset + chunk_size > this->total_size)
|
||||
if (offset + chunk_size > mSize)
|
||||
{
|
||||
chunk_size = total_size - offset;
|
||||
chunk_size = mSize - offset;
|
||||
std::cerr <<"Chunk Size greater than total file size, adjusting chunk " << "size " << chunk_size << std::endl;
|
||||
|
||||
}
|
||||
@ -133,8 +94,6 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data
|
||||
|
||||
long int pos;
|
||||
pos = ftell(fd);
|
||||
std::cerr << pos << " BEFORE RECV SIZE "<< recv_size << std::endl;
|
||||
|
||||
/*
|
||||
* add the data
|
||||
*/
|
||||
@ -144,21 +103,18 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data
|
||||
return 0;
|
||||
}
|
||||
|
||||
this->recv_size += chunk_size;
|
||||
|
||||
pos = ftell(fd);
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileCreator::addFileData() added Data...";
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "recvd: " << recv_size;
|
||||
std::cerr << " pos: " << pos;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
/*
|
||||
* Notify ftFileChunker about chunks received
|
||||
*/
|
||||
fileChunker->notifyReceived(offset,chunk_size);
|
||||
notifyReceived(offset,chunk_size);
|
||||
|
||||
/*
|
||||
* FIXME HANDLE COMPLETION HERE - Any better way?
|
||||
@ -167,6 +123,60 @@ bool ftFileCreator::addFileData(uint64_t offset, uint32_t chunk_size, void *data
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ftFileCreator::initializeFileAttrs()
|
||||
{
|
||||
std::cerr << "ftFileCreator::initializeFileAttrs() Filename: ";
|
||||
std::cerr << file_name;
|
||||
|
||||
/*
|
||||
* check if the file exists
|
||||
*/
|
||||
|
||||
if (ftFileProvider::initializeFileAttrs())
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
if (fd)
|
||||
return 1;
|
||||
|
||||
{
|
||||
std::cout <<
|
||||
"ftFileCreator::initializeFileAttrs() Filename: " << file_name;
|
||||
}
|
||||
|
||||
/*
|
||||
* attempt to open file
|
||||
*/
|
||||
|
||||
fd = fopen(file_name.c_str(), "w+b");
|
||||
if (!fd)
|
||||
{
|
||||
std::cout <<
|
||||
"ftFileCreator::initializeFileAttrs() Failed to open (w+b): "<< file_name << std::endl;
|
||||
return 0;
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* if it opened, find it's length
|
||||
* move to the end
|
||||
*/
|
||||
|
||||
if (0 != fseek(fd, 0L, SEEK_END))
|
||||
{
|
||||
std::cerr << "ftFileCreator::initializeFileAttrs() Seek Failed" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint64_t recvdsize = ftell(fd);
|
||||
|
||||
std::cerr << "ftFileCreator::initializeFileAttrs() File Expected Size: " << mSize << " RecvdSize: " << recvdsize << std::endl;
|
||||
|
||||
return 1;
|
||||
}
|
||||
ftFileCreator::~ftFileCreator()
|
||||
{
|
||||
/*
|
||||
@ -175,546 +185,104 @@ ftFileCreator::~ftFileCreator()
|
||||
}
|
||||
|
||||
|
||||
bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk)
|
||||
int ftFileCreator::notifyReceived(uint64_t offset, uint32_t chunk_size)
|
||||
{
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileCreator::getMissingChunk(???," << chunk << ")";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
return fileChunker->getMissingChunk(offset, chunk);
|
||||
}
|
||||
/* ALREADY LOCKED */
|
||||
|
||||
/***********************************************************
|
||||
*
|
||||
* ftFileChunker methods
|
||||
*
|
||||
***********************************************************/
|
||||
|
||||
ftFileChunker::ftFileChunker(uint64_t size): file_size(size), std_chunk_size(10000), monitorPeriod(30)
|
||||
{
|
||||
/*
|
||||
* srand for randomized version - move it to the sub-class?
|
||||
*/
|
||||
srand ( time(NULL) );
|
||||
aggregate_status = 0;
|
||||
}
|
||||
|
||||
ftFileChunker::~ftFileChunker()
|
||||
{
|
||||
std::vector<ftChunk>::iterator it;
|
||||
for(unsigned int i=0; i<allocationTable.size();i++) {
|
||||
delete allocationTable.at(i); /* Does this need a check? */
|
||||
}
|
||||
if(!allocationTable.empty()){
|
||||
allocationTable.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int ftFileChunker::splitFile(){
|
||||
RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
/*
|
||||
* all in bytes
|
||||
*/
|
||||
num_chunks = file_size/std_chunk_size;
|
||||
/*
|
||||
* FIXME - Remainder should go into last chunk
|
||||
*/
|
||||
uint64_t rem = file_size % std_chunk_size;
|
||||
unsigned int index=0;
|
||||
uint64_t max_chunk_size = file_size - (index * std_chunk_size);
|
||||
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunker::splitFile()";
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tfile_size: " << file_size;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tstd_chunk_size: " << std_chunk_size;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tnum_chunks: " << num_chunks;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\trem: " << rem;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tmax_chunk_size: " << max_chunk_size;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
time_t now = time(NULL);
|
||||
for(index=0;index<num_chunks;index++)
|
||||
/* find the chunk */
|
||||
std::map<uint64_t, ftChunk>::iterator it;
|
||||
it = mChunks.find(offset);
|
||||
if (it == mChunks.end())
|
||||
{
|
||||
uint64_t offset = index * std_chunk_size;
|
||||
max_chunk_size = file_size - (index * std_chunk_size);
|
||||
ftChunk *f = new ftChunk(offset,max_chunk_size,now, ftChunk::AVAIL);
|
||||
allocationTable.push_back(f);
|
||||
return 0; /* ignoring */
|
||||
}
|
||||
|
||||
if (rem != 0)
|
||||
ftChunk chunk = it->second;
|
||||
mChunks.erase(it);
|
||||
|
||||
if (chunk.chunk != chunk_size)
|
||||
{
|
||||
ftChunk *f = new ftChunk(file_size-rem,rem,now, ftChunk::AVAIL);
|
||||
allocationTable.push_back(f);
|
||||
num_chunks++;
|
||||
/* partial : shrink chunk */
|
||||
chunk.chunk -= chunk_size;
|
||||
chunk.offset += chunk_size;
|
||||
mChunks[chunk.offset] = chunk;
|
||||
}
|
||||
|
||||
/*
|
||||
* DEBUGGER
|
||||
* for(int j=0;j<allocationTable.size();j++)
|
||||
* {
|
||||
* std::cout << "SIZE " << allocationTable.at(j)->max_chunk_size << " " << allocationTable.at(j)->chunk_status << std::endl;
|
||||
* }
|
||||
*/
|
||||
/* if we've cleaned up chunks, must update counters */
|
||||
it = mChunks.find(chunk.offset);
|
||||
if (it == mChunks.end())
|
||||
{
|
||||
mStart = mEnd;
|
||||
}
|
||||
else if (it == mChunks.begin())
|
||||
{
|
||||
mStart += it->second.offset;
|
||||
}
|
||||
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/*
|
||||
* This method sets the offset, chunk may be reset if needed
|
||||
*/
|
||||
bool ftFileCreator::getMissingChunk(uint64_t &offset, uint32_t &chunk)
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
std::cerr << "ffc::getMissingChunk(...,"<< chunk << ")"<< std::endl;
|
||||
|
||||
bool ftFileChunker::getMissingChunk(uint64_t &offset, uint32_t &chunk)
|
||||
{
|
||||
RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/
|
||||
/* check start point */
|
||||
|
||||
unsigned int i =0;
|
||||
bool found = false;
|
||||
int chunks_after = 0;
|
||||
int chunks_rem = 0;
|
||||
if (mStart == mSize)
|
||||
{
|
||||
std::cerr << "ffc::getMissingChunk() File Done";
|
||||
std::cerr << std::endl;
|
||||
return false;
|
||||
}
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunker::getMissingChunk(???," << chunk << ")";
|
||||
/* check for freed chunks */
|
||||
time_t ts = time(NULL);
|
||||
time_t old = ts-CHUNK_MAX_AGE;
|
||||
|
||||
std::map<uint64_t, ftChunk>::iterator it;
|
||||
for(it = mChunks.begin(); it != mChunks.end(); it++)
|
||||
{
|
||||
/* very simple algorithm */
|
||||
if (it->second.ts < old)
|
||||
{
|
||||
std::cerr << "ffc::getMissingChunk() ReAlloc";
|
||||
std::cerr << std::endl;
|
||||
|
||||
/* retry this one */
|
||||
it->second.ts = ts;
|
||||
chunk = it->second.chunk;
|
||||
offset = it->second.offset;
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
std::cerr << "ffc::getMissingChunk() new Alloc";
|
||||
std::cerr << " mStart: " << mStart << " mEnd: " << mEnd;
|
||||
std::cerr << "mSize: " << mSize;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
/*
|
||||
* This signals file completion
|
||||
* FIXME Does it need to be more explicit
|
||||
*/
|
||||
|
||||
if(aggregate_status == num_chunks * ftChunk::RECEIVED)
|
||||
{
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "completed ??";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
return found;
|
||||
}
|
||||
/* else allocate a new chunk */
|
||||
if (mSize - mEnd < chunk)
|
||||
chunk = mSize - mEnd;
|
||||
|
||||
offset = mEnd;
|
||||
mEnd += chunk;
|
||||
|
||||
while(i<allocationTable.size())
|
||||
{
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunker checking allocTable(" << i << ")";
|
||||
std::cerr << " of " << allocationTable.size();
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
if(allocationTable.at(i)->max_chunk_size >=chunk)
|
||||
{
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunker pre alloc(" << i << ")";
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\toffset: " << allocationTable.at(i)->offset;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tmax_chunk: " << allocationTable.at(i)->max_chunk_size;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\treq_chunk: " << chunk;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
|
||||
offset = allocationTable.at(i)->offset;
|
||||
chunks_after = chunk/std_chunk_size; //10KB
|
||||
|
||||
/*
|
||||
* FIXME Handling remaining chunk < 10KB
|
||||
*/
|
||||
|
||||
//if (chunk <
|
||||
chunks_rem = chunk % std_chunk_size;
|
||||
chunk -= chunks_rem;
|
||||
/*std::cout << "Found " << chunk << " at " << i << " "<< chunks_after << std::endl;*/
|
||||
allocationTable.at(i)->max_chunk_size=0;
|
||||
allocationTable.at(i)->timestamp = time(NULL);
|
||||
allocationTable.at(i)->chunk_status = ftChunk::ALLOCATED;
|
||||
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunker postalloc(" << i << ")";
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tchunks_after: " << chunks_after;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tchunks_rem: " << chunks_after;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\tchunk: " << chunks_after;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
|
||||
/* if we get here, there is no available chunk bigger
|
||||
* than requested ...
|
||||
* NB: Request size should be a larger than std_chunk_size.
|
||||
* So Edge (sub chunk allocation) condition is handled here.
|
||||
*
|
||||
* Find largest remaining chunk.
|
||||
*/
|
||||
|
||||
if (!found)
|
||||
{
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunker::getMissingChuck() FULL CHUNK not found";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
i=0;
|
||||
uint64_t max = allocationTable.at(i)->max_chunk_size;
|
||||
uint64_t size = max;
|
||||
int maxi = -1;
|
||||
if (max > 0)
|
||||
{
|
||||
maxi = 0;
|
||||
}
|
||||
|
||||
while(i<allocationTable.size())
|
||||
{
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "Checking(" << i << ") " <<
|
||||
allocationTable.at(i)->max_chunk_size;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
size = allocationTable.at(i)->max_chunk_size;
|
||||
if(size > max)
|
||||
{
|
||||
max = allocationTable.at(i)->max_chunk_size;
|
||||
maxi = i;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
if (maxi > -1) //maxi or max
|
||||
{
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "Biggest Avail Chunk: " << max;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
offset = allocationTable.at(maxi)->offset;
|
||||
chunk = allocationTable.at(maxi)->max_chunk_size;
|
||||
chunks_after = chunk/std_chunk_size; //10KB
|
||||
|
||||
/* Handle end condition ...
|
||||
* max_chunk_size < std_chunk_size
|
||||
* Trim if not end condition.
|
||||
*/
|
||||
if (chunks_after > 0)
|
||||
{
|
||||
chunks_rem = chunk % std_chunk_size;
|
||||
chunk -= chunks_rem;
|
||||
}
|
||||
else
|
||||
{
|
||||
/* end condition */
|
||||
chunks_after = 1;
|
||||
}
|
||||
|
||||
allocationTable.at(maxi)->max_chunk_size=0;
|
||||
allocationTable.at(maxi)->timestamp = time(NULL);
|
||||
allocationTable.at(maxi)->chunk_status = ftChunk::ALLOCATED;
|
||||
found = true;
|
||||
i = maxi;
|
||||
}
|
||||
|
||||
} //if not found
|
||||
|
||||
|
||||
if (found)
|
||||
{
|
||||
// i represents index to chunk(s) we will use.
|
||||
// chunks_after is the count of how many we will use.
|
||||
|
||||
std::cout << "Chunks remaining " << chunks_rem << std::endl;
|
||||
/*
|
||||
* update all previous chunks max available size
|
||||
* Expensive? Can it be smarter FIXME
|
||||
*/
|
||||
|
||||
/* drbob: Think this is wrong?
|
||||
* disabling...
|
||||
*
|
||||
for(unsigned int j=0;j<i;j++)
|
||||
{
|
||||
if (allocationTable.at(j)->max_chunk_size >0)
|
||||
allocationTable.at(j)->max_chunk_size -= chunk;
|
||||
}
|
||||
*
|
||||
*/
|
||||
|
||||
|
||||
for(unsigned int j=i;j<i+chunks_after;j++)
|
||||
{
|
||||
allocationTable.at(j)->max_chunk_size = 0;
|
||||
allocationTable.at(j)->chunk_status = ftChunk::ALLOCATED;
|
||||
}
|
||||
|
||||
// DEBUGGER - Uncomment
|
||||
for(unsigned int j=0;j<allocationTable.size();j++)
|
||||
{
|
||||
std::cout << "After allocation " << allocationTable.at(j)->max_chunk_size << " " << allocationTable.at(j)->chunk_status << std::endl;
|
||||
}
|
||||
|
||||
}
|
||||
else
|
||||
{
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunker::getMissingChuck() not found";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
return found;
|
||||
mChunks[offset] = ftChunk(offset, chunk, ts);
|
||||
return true;
|
||||
}
|
||||
|
||||
/*
|
||||
* This should run on a separate thread when ftFileChunker is initialized
|
||||
* FIXME Implemet DrBob's suggestion of request-fired check instead of dedicated
|
||||
* thread
|
||||
*/
|
||||
int ftFileChunker::monitor()
|
||||
{
|
||||
int reset = 0;
|
||||
uint32_t prev_size = 0;
|
||||
uint32_t size = 0;
|
||||
|
||||
std::cout<<"Running monitor.."<<std::endl;
|
||||
|
||||
RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/
|
||||
for(unsigned int j=allocationTable.size()-1 ;j>= 0;)
|
||||
{
|
||||
if((allocationTable.at(j)->chunk_status == ftChunk::ALLOCATED) &&
|
||||
(allocationTable.at(j)->timestamp - time(NULL) > 30))
|
||||
{
|
||||
allocationTable.at(j)->chunk_status = ftChunk::AVAIL;
|
||||
if (j == allocationTable.size()-1)
|
||||
{
|
||||
/* at end */
|
||||
prev_size = 0;
|
||||
size = file_size % std_chunk_size;
|
||||
if (size == 0)
|
||||
{
|
||||
size = std_chunk_size;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
prev_size = allocationTable.at(j+1)->max_chunk_size;
|
||||
size = std_chunk_size;
|
||||
}
|
||||
|
||||
allocationTable.at(j)->max_chunk_size = size + prev_size;
|
||||
prev_size = allocationTable.at(j)->max_chunk_size;
|
||||
|
||||
for(j--; j >= 0; j--)
|
||||
{
|
||||
if (allocationTable.at(j)->chunk_status != ftChunk::AVAIL)
|
||||
break;
|
||||
|
||||
allocationTable.at(j)->max_chunk_size += prev_size;
|
||||
prev_size = allocationTable.at(j)->max_chunk_size;
|
||||
reset++;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
j--;
|
||||
}
|
||||
}
|
||||
return reset;
|
||||
}
|
||||
|
||||
void ftFileChunker::setMonitorPeriod(int period)
|
||||
{
|
||||
monitorPeriod = period;
|
||||
}
|
||||
|
||||
void ftFileChunker::run()
|
||||
{
|
||||
|
||||
while(1)
|
||||
{
|
||||
|
||||
for(int i = 0; i < monitorPeriod; i++)
|
||||
{
|
||||
|
||||
/******************** WINDOWS/UNIX SPECIFIC PART ******************/
|
||||
#ifndef WINDOWS_SYS
|
||||
sleep(1);
|
||||
#else
|
||||
|
||||
Sleep(1000);
|
||||
#endif
|
||||
/******************* WINDOWS/UNIX SPECIFIC PART ******************/
|
||||
}
|
||||
monitor();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int ftFileChunker::notifyReceived(uint64_t offset, uint32_t chunk_size)
|
||||
{
|
||||
RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/
|
||||
int index = offset / std_chunk_size;
|
||||
|
||||
/* should receive a multiple of chunk_size.... */
|
||||
uint32_t chunks = chunk_size / std_chunk_size;
|
||||
uint32_t rem_chunk = chunk_size % std_chunk_size;
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunkerr::notifyReceived(";
|
||||
std::cerr << offset;
|
||||
std::cerr << ", " << chunk_size << ")";
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\t# complete chunks: " << chunks;
|
||||
std::cerr << std::endl;
|
||||
std::cerr << "\trem_chunk: " << rem_chunk;
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
if(allocationTable.at(index)->chunk_status == ftChunk::ALLOCATED){
|
||||
allocationTable.at(index)->chunk_status = ftChunk::RECEIVED;
|
||||
aggregate_status += ftChunk::RECEIVED;
|
||||
|
||||
#ifdef FILE_DEBUG
|
||||
std::cerr << "ftFileChunkerr::notifyReceived()";
|
||||
std::cerr << " flagged as RECVD";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
|
||||
}
|
||||
return aggregate_status;
|
||||
}
|
||||
|
||||
/***********************************************************
|
||||
*
|
||||
* ftFileRandomizeChunker methods
|
||||
*
|
||||
***********************************************************/
|
||||
|
||||
ftFileRandomizeChunker::ftFileRandomizeChunker(uint64_t size):
|
||||
ftFileChunker(size)
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
ftFileRandomizeChunker::~ftFileRandomizeChunker()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
bool ftFileRandomizeChunker::getMissingChunk(uint64_t &offset, uint32_t &chunk) {
|
||||
|
||||
RsStackMutex stack(chunkerMutex); /********** STACK LOCKED MTX ******/
|
||||
std::cerr << "Calling getMissingChunk with chunk="<< chunk << std::endl;
|
||||
unsigned int i =0;
|
||||
bool found = false;
|
||||
int chunks_after = 0;
|
||||
int chunks_rem = 0;
|
||||
|
||||
if(aggregate_status == num_chunks * ftChunk::RECEIVED)
|
||||
return found;
|
||||
|
||||
std::vector<int> randomIndex;
|
||||
while(i<allocationTable.size())
|
||||
{
|
||||
if(allocationTable.at(i)->max_chunk_size >=chunk){
|
||||
randomIndex.push_back(i);
|
||||
}
|
||||
i++;
|
||||
}
|
||||
|
||||
/*
|
||||
* FIXME test sufficiently to make sure its picking every index
|
||||
*/
|
||||
if (randomIndex.size()>0)
|
||||
{
|
||||
int rnum = rand() % randomIndex.size();
|
||||
i = randomIndex.at(rnum);
|
||||
std::cout << "i=" <<i << " rnum " << rnum << std::endl;
|
||||
offset = allocationTable.at(i)->offset;
|
||||
chunks_after = chunk/std_chunk_size; //10KB
|
||||
chunks_rem = chunk % std_chunk_size;
|
||||
chunk -= chunks_rem;
|
||||
std::cout << "Found " << chunk << " at index =" << i << " "<< chunks_after << std::endl;
|
||||
allocationTable.at(i)->max_chunk_size=0;
|
||||
allocationTable.at(i)->timestamp = time(NULL);
|
||||
allocationTable.at(i)->chunk_status = ftChunk::ALLOCATED;
|
||||
found = true;
|
||||
}
|
||||
|
||||
if (!found)
|
||||
{
|
||||
i=0;
|
||||
uint64_t min = allocationTable.at(i)->max_chunk_size - chunk;
|
||||
uint64_t diff = min;
|
||||
int mini = -1;
|
||||
while(i<allocationTable.size())
|
||||
{
|
||||
diff = allocationTable.at(i)->max_chunk_size-chunk;
|
||||
if(diff <= min && diff >0)
|
||||
{
|
||||
min = allocationTable.at(i)->max_chunk_size - chunk;
|
||||
mini = i;
|
||||
}
|
||||
i++;
|
||||
}
|
||||
if (mini > -1)
|
||||
{
|
||||
offset = allocationTable.at(mini)->offset;
|
||||
chunk = allocationTable.at(mini)->max_chunk_size;
|
||||
chunks_after = chunk/std_chunk_size; //10KB
|
||||
chunks_rem = chunk % std_chunk_size;
|
||||
chunk -= chunks_rem;
|
||||
allocationTable.at(mini)->max_chunk_size=0;
|
||||
allocationTable.at(mini)->timestamp = time(NULL);
|
||||
allocationTable.at(mini)->chunk_status = ftChunk::ALLOCATED;
|
||||
found = true;
|
||||
}
|
||||
|
||||
} //if not found
|
||||
|
||||
if (found)
|
||||
{
|
||||
// update all previous chunks max available size
|
||||
for(unsigned int j=0;j<i;j++)
|
||||
{
|
||||
if (allocationTable.at(j)->max_chunk_size >0)
|
||||
allocationTable.at(j)->max_chunk_size -= chunk;
|
||||
}
|
||||
|
||||
for(unsigned int j=i;j<i+chunks_after;j++)
|
||||
{
|
||||
allocationTable.at(j)->max_chunk_size = 0;
|
||||
allocationTable.at(j)->chunk_status = ftChunk::ALLOCATED;
|
||||
}
|
||||
|
||||
/* for(int j=0;j<allocationTable.size();j++){
|
||||
std::cout << "After allocation " << j << " " << allocationTable.at(j)->max_chunk_size << " " << allocationTable.at(j)->chunk_status << std::endl;
|
||||
}*/
|
||||
}
|
||||
return found;
|
||||
}
|
||||
/***********************************************************
|
||||
*
|
||||
* ftChunk methods
|
||||
*
|
||||
***********************************************************/
|
||||
|
||||
ftChunk::ftChunk(uint64_t offset,uint64_t chunk_size,time_t time, ftChunk::Status s) : offset(offset), max_chunk_size(chunk_size), timestamp(time), chunk_status(s)
|
||||
ftChunk::ftChunk(uint64_t ioffset,uint64_t size,time_t now)
|
||||
: offset(ioffset), chunk(size), ts(now)
|
||||
{
|
||||
|
||||
}
|
||||
@ -723,3 +291,4 @@ ftChunk::~ftChunk()
|
||||
{
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -33,100 +33,56 @@
|
||||
*
|
||||
*/
|
||||
#include "ftfileprovider.h"
|
||||
#include "util/rsthreads.h"
|
||||
#include <queue>
|
||||
#include <map>
|
||||
class ftChunk;
|
||||
class ftFileChunker;
|
||||
|
||||
class ftFileCreator: public ftFileProvider
|
||||
{
|
||||
public:
|
||||
|
||||
ftFileCreator(std::string savepath, uint64_t size, std::string hash,std::string chunker);
|
||||
ftFileCreator(std::string savepath, uint64_t size, std::string hash, uint64_t recvd);
|
||||
|
||||
~ftFileCreator();
|
||||
|
||||
/*
|
||||
* overloaded from FileProvider FIXME
|
||||
* virtual bool getFileData(uint64_t offset, uint32_t chunk_size, void *data);
|
||||
*
|
||||
*/
|
||||
/* overloaded from FileProvider */
|
||||
virtual bool getFileData(uint64_t offset, uint32_t chunk_size, void *data);
|
||||
uint64_t getRecvd();
|
||||
|
||||
/*
|
||||
* FIXME: initializeFileAttrs
|
||||
* Should this be a over-ridden version of ftfile provider?
|
||||
* What happens in the case of simultaneous upload and download?
|
||||
*/
|
||||
|
||||
int initializeFileAttrs();
|
||||
/*
|
||||
* creation functions for FileCreator
|
||||
*/
|
||||
|
||||
bool getMissingChunk(uint64_t &offset, uint32_t &chunk);
|
||||
bool addFileData(uint64_t offset, uint32_t chunk_size, void *data);
|
||||
|
||||
protected:
|
||||
|
||||
virtual int initializeFileAttrs();
|
||||
|
||||
private:
|
||||
|
||||
int notifyReceived(uint64_t offset, uint32_t chunk_size);
|
||||
/*
|
||||
* structure to track missing chunks
|
||||
*/
|
||||
|
||||
uint64_t recv_size;
|
||||
std::string hash;
|
||||
ftFileChunker *fileChunker;
|
||||
void initialize(std::string);
|
||||
RsMutex ftcMutex;
|
||||
};
|
||||
uint64_t mStart;
|
||||
uint64_t mEnd;
|
||||
|
||||
/*
|
||||
This class can either be specialized to follow a different splitting
|
||||
policy or have an argument to indicate different policies
|
||||
*/
|
||||
class ftFileChunker : public RsThread {
|
||||
public:
|
||||
/*
|
||||
* FIXME Does filechunker require hash??
|
||||
*/
|
||||
ftFileChunker(uint64_t size);
|
||||
virtual ~ftFileChunker();
|
||||
|
||||
/*
|
||||
* Breaks up the file into evenly sized chunks
|
||||
* Initializes all chunks to never_requested
|
||||
*/
|
||||
int splitFile();
|
||||
virtual void run();
|
||||
virtual bool getMissingChunk(uint64_t &offset, uint32_t &chunk);
|
||||
bool getMissingChunkRandom(uint64_t &offset, uint32_t &chunk);
|
||||
int notifyReceived(uint64_t offset, uint32_t chunk_size);
|
||||
int monitor();
|
||||
void setMonitorPeriod(int period);
|
||||
protected:
|
||||
uint64_t file_size;
|
||||
uint64_t num_chunks;
|
||||
uint64_t std_chunk_size;
|
||||
std::vector<ftChunk*> allocationTable;
|
||||
unsigned int aggregate_status;
|
||||
int monitorPeriod;
|
||||
RsMutex chunkerMutex; /********** STACK LOCKED MTX ******/
|
||||
};
|
||||
|
||||
class ftFileRandomizeChunker : public ftFileChunker {
|
||||
public:
|
||||
ftFileRandomizeChunker(uint64_t size);
|
||||
virtual bool getMissingChunk(uint64_t &offset, uint32_t &chunk);
|
||||
~ftFileRandomizeChunker();
|
||||
std::map<uint64_t, ftChunk> mChunks;
|
||||
|
||||
};
|
||||
|
||||
class ftChunk {
|
||||
public:
|
||||
enum Status {AVAIL, ALLOCATED, RECEIVED};
|
||||
ftChunk(uint64_t,uint64_t,time_t,Status);
|
||||
uint64_t offset;
|
||||
uint64_t max_chunk_size;
|
||||
time_t timestamp;
|
||||
Status chunk_status;
|
||||
ftChunk(uint64_t,uint64_t,time_t);
|
||||
ftChunk():offset(0), chunk(0), ts(0) {}
|
||||
~ftChunk();
|
||||
|
||||
uint64_t offset;
|
||||
uint64_t chunk;
|
||||
time_t ts;
|
||||
};
|
||||
|
||||
#endif // FT_FILE_PROVIDER_HEADER
|
||||
#endif // FT_FILE_CREATOR_HEADER
|
||||
|
||||
@ -1,87 +1,89 @@
|
||||
#include "ftfilecreator.h"
|
||||
|
||||
main(){
|
||||
uint64_t offset;
|
||||
uint32_t csize;
|
||||
/*Test file creator*/
|
||||
ftFileCreator fcreator("somefile",100000,"hash","default");
|
||||
csize = 40000;
|
||||
if (fcreator.getMissingChunk(offset, csize)){
|
||||
std::cout << "Missing Chunk's offset=" << offset << " chunk_size=" << csize << std::endl;
|
||||
}
|
||||
|
||||
csize = 10000;
|
||||
if (fcreator.getMissingChunk(offset, csize)){
|
||||
std::cout << "Missing Chunk's offset=" << offset << " chunk_size=" << csize << std::endl;
|
||||
}
|
||||
|
||||
csize = 15000;
|
||||
if (fcreator.getMissingChunk(offset, csize)){
|
||||
std::cout << "Missing Chunk's offset=" << offset << " chunk_size=" << csize << std::endl;
|
||||
}
|
||||
|
||||
|
||||
std::cout << "Test file creator adding data to file out-of-order\n";
|
||||
char* alpha = "abcdefghij";
|
||||
std::cerr << "Call to addFileData =" << fcreator.addFileData(10,10,alpha );
|
||||
char* numerical = "1234567890";
|
||||
std::cerr << "Call to addFileData =" << fcreator.addFileData(0,10,numerical );
|
||||
|
||||
#include "util/utest.h"
|
||||
|
||||
/* Test file creator can write out of order/randomized chunker */
|
||||
ftFileCreator franc("somefile1",50000,"hash", "randomize");
|
||||
csize = 30000;
|
||||
if (franc.getMissingChunk(offset, csize)){
|
||||
std::cout << "Offset " << offset << " Csize " << csize << std::endl;
|
||||
}
|
||||
INITTEST();
|
||||
|
||||
char* allA = (char*) malloc(csize);
|
||||
for(int i=0;i<csize;i++)
|
||||
allA[i]='a';
|
||||
|
||||
std::cerr << "ADD DATA RETUNED " << franc.addFileData(offset,csize,allA );
|
||||
static int test_timeout(ftFileCreator *creator);
|
||||
static int test_fill(ftFileCreator *creator);
|
||||
|
||||
csize = 10000;
|
||||
if (franc.getMissingChunk(offset, csize)){
|
||||
std::cout << "Offset " << offset << " Csize " << csize << std::endl;
|
||||
}
|
||||
|
||||
char* allB = (char*) malloc(csize);
|
||||
for(int i=0;i<csize;i++)
|
||||
allB[i]='b';
|
||||
int main()
|
||||
{
|
||||
/* use ftcreator to create a file on tmp drive */
|
||||
ftFileCreator fcreator("/tmp/rs-ftfc-test.dta",100000,"hash",0);
|
||||
|
||||
test_timeout(&fcreator);
|
||||
test_fill(&fcreator);
|
||||
|
||||
FINALREPORT("RsTlvItem Stack Tests");
|
||||
|
||||
std::cerr << "ADD DATA RETUNED " << franc.addFileData(offset,csize,allB );
|
||||
|
||||
if (franc.getMissingChunk(offset, csize)){
|
||||
std::cout << "Offset " << offset << " Csize " << csize << std::endl;
|
||||
}
|
||||
else {
|
||||
std::cout << "getMissing chunk returned nothing" << std::endl;
|
||||
}
|
||||
|
||||
|
||||
csize = 10000;
|
||||
if (franc.getMissingChunk(offset, csize)){
|
||||
std::cout << "Offset " << offset << " Csize " << csize << std::endl;
|
||||
}
|
||||
|
||||
char* allC = (char*) malloc(csize);
|
||||
for(int i=0;i<csize;i++)
|
||||
allC[i]='c';
|
||||
|
||||
|
||||
|
||||
std::cerr << "ADD DATA RETUNED " << franc.addFileData(offset,csize,allC );
|
||||
|
||||
if (franc.getMissingChunk(offset, csize)){
|
||||
std::cout << "Offset " << offset << " Csize " << csize << std::endl;
|
||||
}
|
||||
else {
|
||||
std::cout << "getMissing chunk returned nothing" << std::endl;
|
||||
}
|
||||
|
||||
free(allA);
|
||||
free(allB);
|
||||
return TESTRESULT();
|
||||
}
|
||||
|
||||
|
||||
int test_timeout(ftFileCreator *creator)
|
||||
{
|
||||
uint32_t chunk = 1000;
|
||||
uint64_t offset = 0;
|
||||
int max_timeout = 30;
|
||||
int max_offset = chunk * max_timeout;
|
||||
int i;
|
||||
std::cerr << "60 second test of chunk queue.";
|
||||
std::cerr << std::endl;
|
||||
|
||||
for(i = 0; i < max_timeout; i++)
|
||||
{
|
||||
creator->getMissingChunk(offset, chunk);
|
||||
std::cerr << "Allocated Offset: " << offset << " chunk: " << chunk << std::endl;
|
||||
|
||||
CHECK(offset < max_offset);
|
||||
sleep(1);
|
||||
}
|
||||
|
||||
std::cerr << "Expect Repeats now";
|
||||
std::cerr << std::endl;
|
||||
|
||||
for(i = 0; i < max_timeout; i++)
|
||||
{
|
||||
creator->getMissingChunk(offset, chunk);
|
||||
std::cerr << "Allocated Offset: " << offset << " chunk: " << chunk << std::endl;
|
||||
|
||||
CHECK(offset < max_offset);
|
||||
sleep(1);
|
||||
}
|
||||
REPORT("Chunk Queue");
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
int test_fill(ftFileCreator *creator)
|
||||
{
|
||||
uint32_t chunk = 1000;
|
||||
uint64_t offset = 0;
|
||||
|
||||
uint64_t init_size = creator->getFileSize();
|
||||
uint64_t init_trans = creator->getRecvd();
|
||||
std::cerr << "Initial FileSize: " << init_size << std::endl;
|
||||
std::cerr << "Initial Transferred:" << init_trans << std::endl;
|
||||
|
||||
while(creator->getMissingChunk(offset, chunk))
|
||||
{
|
||||
/* give it too them */
|
||||
void *data = malloc(chunk);
|
||||
creator->addFileData(offset, chunk, data);
|
||||
free(data);
|
||||
}
|
||||
|
||||
uint64_t end_size = creator->getFileSize();
|
||||
uint64_t end_trans = creator->getRecvd();
|
||||
std::cerr << "End FileSize: " << end_size << std::endl;
|
||||
std::cerr << "End Transferred:" << end_trans << std::endl;
|
||||
CHECK(init_size == end_size);
|
||||
CHECK(end_trans == end_size);
|
||||
|
||||
REPORT("Test Fill");
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
|
||||
@ -3,8 +3,8 @@
|
||||
#include "util/rsdir.h"
|
||||
|
||||
ftFileProvider::ftFileProvider(std::string path, uint64_t size, std::string
|
||||
hash) : total_size(size), hash(hash), file_name(path), fd(NULL) {
|
||||
|
||||
hash) : mSize(size), hash(hash), file_name(path), fd(NULL)
|
||||
{
|
||||
}
|
||||
|
||||
ftFileProvider::~ftFileProvider(){
|
||||
@ -13,20 +13,29 @@ ftFileProvider::~ftFileProvider(){
|
||||
}
|
||||
}
|
||||
|
||||
bool ftFileProvider::fileOk()
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
return (fd != NULL);
|
||||
}
|
||||
|
||||
std::string ftFileProvider::getHash()
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
return hash;
|
||||
}
|
||||
|
||||
uint64_t ftFileProvider::getFileSize()
|
||||
{
|
||||
return total_size;
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
return mSize;
|
||||
}
|
||||
|
||||
bool ftFileProvider::FileDetails(FileInfo &info)
|
||||
{
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
info.hash = hash;
|
||||
info.size = total_size;
|
||||
info.size = mSize;
|
||||
info.path = file_name;
|
||||
info.fname = RsDirUtil::getTopDir(file_name);
|
||||
|
||||
@ -38,17 +47,15 @@ bool ftFileProvider::FileDetails(FileInfo &info)
|
||||
|
||||
bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *data)
|
||||
{
|
||||
RsStackMutex stack(ftPMutex); /********** STACK LOCKED MTX ******/
|
||||
if (fd==NULL)
|
||||
{
|
||||
int init = initializeFileAttrs();
|
||||
if (init ==0)
|
||||
{
|
||||
std::cerr <<"Initialization Failed" << std::endl;
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* dodgey checking outside of mutex...
|
||||
* much check again inside FileAttrs().
|
||||
*/
|
||||
if (fd == NULL)
|
||||
if (!initializeFileAttrs())
|
||||
return false;
|
||||
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
|
||||
/*
|
||||
* Use private data, which has a pointer to file, total size etc
|
||||
*/
|
||||
@ -59,9 +66,9 @@ bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *dat
|
||||
int data_size = chunk_size;
|
||||
long base_loc = offset;
|
||||
|
||||
if (base_loc + data_size > total_size)
|
||||
if (base_loc + data_size > mSize)
|
||||
{
|
||||
data_size = total_size - base_loc;
|
||||
data_size = mSize - base_loc;
|
||||
std::cerr <<"Chunk Size greater than total file size, adjusting chunk size " << data_size << std::endl;
|
||||
}
|
||||
|
||||
@ -108,6 +115,14 @@ bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *dat
|
||||
|
||||
int ftFileProvider::initializeFileAttrs()
|
||||
{
|
||||
std::cerr << "ftFileProvider::initializeFileAttrs() Filename: ";
|
||||
std::cerr << file_name;
|
||||
|
||||
|
||||
RsStackMutex stack(ftcMutex); /********** STACK LOCKED MTX ******/
|
||||
if (fd)
|
||||
return 1;
|
||||
|
||||
/*
|
||||
* check if the file exists
|
||||
*/
|
||||
@ -130,7 +145,6 @@ int ftFileProvider::initializeFileAttrs()
|
||||
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* if it opened, find it's length
|
||||
* move to the end
|
||||
@ -142,6 +156,9 @@ int ftFileProvider::initializeFileAttrs()
|
||||
return 0;
|
||||
}
|
||||
|
||||
total_size = ftell(fd);
|
||||
uint64_t recvdsize = ftell(fd);
|
||||
|
||||
std::cerr << "ftFileProvider::initializeFileAttrs() File Expected Size: " << mSize << " RecvdSize: " << recvdsize << std::endl;
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -40,15 +40,18 @@ class ftFileProvider
|
||||
public:
|
||||
ftFileProvider(std::string path, uint64_t size, std::string hash);
|
||||
virtual ~ftFileProvider();
|
||||
|
||||
virtual bool getFileData(uint64_t offset, uint32_t chunk_size, void *data);
|
||||
virtual bool FileDetails(FileInfo &info);
|
||||
std::string getHash();
|
||||
uint64_t getFileSize();
|
||||
bool fileOk();
|
||||
|
||||
|
||||
protected:
|
||||
virtual int initializeFileAttrs();
|
||||
uint64_t total_size;
|
||||
virtual int initializeFileAttrs(); /* does for both */
|
||||
|
||||
uint64_t mSize;
|
||||
std::string hash;
|
||||
std::string file_name;
|
||||
FILE *fd;
|
||||
@ -65,7 +68,7 @@ protected:
|
||||
/*
|
||||
* Mutex Required for stuff below
|
||||
*/
|
||||
RsMutex ftPMutex;
|
||||
RsMutex ftcMutex;
|
||||
};
|
||||
|
||||
|
||||
|
||||
@ -1,33 +1,72 @@
|
||||
#include "ftfileprovider.h"
|
||||
#include "ftfilecreator.h"
|
||||
|
||||
int main(){
|
||||
ftFileProvider fp("dummy.txt",1,"ABCDEF");
|
||||
char data[2];
|
||||
long offset = 0;
|
||||
for (int i=0;i<10;i++) {
|
||||
|
||||
if (fp.getFileData(offset,2,&data)){
|
||||
std::cout <<"Recv data " << data[0] << std::endl;
|
||||
#include "util/utest.h"
|
||||
|
||||
|
||||
INITTEST()
|
||||
|
||||
int main()
|
||||
{
|
||||
|
||||
/* create a random file */
|
||||
uint64_t size = 100000;
|
||||
uint32_t max_chunk = 10000;
|
||||
uint32_t chunk = 1000;
|
||||
uint64_t offset = 0;
|
||||
|
||||
std::string filename = "/tmp/ft_test.dta";
|
||||
std::string filename2 = "/tmp/ft_test.dta.dup";
|
||||
|
||||
/* use creator to make it */
|
||||
|
||||
void *data = malloc(max_chunk);
|
||||
|
||||
ftFileCreator *creator = new ftFileCreator(filename, size, "hash", 0);
|
||||
for(offset = 0; offset != size; offset += chunk)
|
||||
{
|
||||
if (!creator->addFileData(offset, chunk, data))
|
||||
{
|
||||
FAILED("Create Test Data File");
|
||||
std::cerr << "Failed to add data (CREATE)";
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
else {
|
||||
std::cout <<"Recv no data." << std::endl;
|
||||
}
|
||||
offset+=2;
|
||||
}
|
||||
delete creator;
|
||||
|
||||
|
||||
ftFileProvider fp1("dummy1.txt",3,"ABCDEF");
|
||||
char data1[3];
|
||||
offset = 0;
|
||||
for (int i=0;i<10;i++) {
|
||||
|
||||
if (fp1.getFileData(offset,2,&data1)){
|
||||
std::cout <<"Recv data " << data1[0] << std::endl;
|
||||
std::cerr << "Created file: " << filename << " of size: " << size;
|
||||
std::cerr << std::endl;
|
||||
|
||||
/* load it with file provider */
|
||||
creator = new ftFileCreator(filename2, size, "hash", 0);
|
||||
ftFileProvider *provider = new ftFileProvider(filename, size, "hash");
|
||||
|
||||
/* create duplicate with file creator */
|
||||
|
||||
while(creator->getMissingChunk(offset, chunk))
|
||||
{
|
||||
if (!provider->getFileData(offset, chunk, data))
|
||||
{
|
||||
FAILED("Read from Test Data File");
|
||||
std::cerr << "Failed to get data";
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
else {
|
||||
std::cout <<"Revc no data" << std::endl;
|
||||
|
||||
if (!creator->addFileData(offset, chunk, data))
|
||||
{
|
||||
FAILED("Write to Duplicate");
|
||||
std::cerr << "Failed to add data";
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
offset+=2;
|
||||
|
||||
std::cerr << "Transferred: " << chunk << " @ " << offset;
|
||||
std::cerr << std::endl;
|
||||
|
||||
|
||||
/* reset chunk size */
|
||||
chunk = (uint64_t) max_chunk * (rand() / (1.0 + RAND_MAX));
|
||||
|
||||
std::cerr << "ChunkSize = " << chunk << std::endl;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
@ -140,16 +140,35 @@ bool ftTransferModule::recvFileData(std::string peerId, uint64_t offset,
|
||||
mit = mFileSources.find(peerId);
|
||||
|
||||
if (mit == mFileSources.end())
|
||||
return false;
|
||||
{
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::recvFileData()";
|
||||
std::cerr << " peer not found in sources";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
if ((mit->second).state != PQIPEER_DOWNLOADING)
|
||||
return false;
|
||||
{
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::recvFileData()";
|
||||
std::cerr << " peer not downloading???";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
//return false;
|
||||
}
|
||||
|
||||
if (offset != ((mit->second).offset + (mit->second).receivedSize))
|
||||
{
|
||||
//fix me
|
||||
//received data not expected
|
||||
return false;
|
||||
#ifdef FT_DEBUG
|
||||
std::cerr << "ftTransferModule::recvFileData()";
|
||||
std::cerr << " offset != offset + recvdSize";
|
||||
std::cerr << std::endl;
|
||||
#endif
|
||||
return false;
|
||||
}
|
||||
|
||||
(mit->second).receivedSize += chunk_size;
|
||||
|
||||
180
libretroshare/src/ft/fttransfermoduletest.cc
Normal file
180
libretroshare/src/ft/fttransfermoduletest.cc
Normal file
@ -0,0 +1,180 @@
|
||||
/*
|
||||
* libretroshare/src/ft: fttransfermoduletest.cc
|
||||
*
|
||||
* File Transfer for RetroShare.
|
||||
*
|
||||
* Copyright 2008 by Robert Fernie.
|
||||
*
|
||||
* This library is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU Library General Public
|
||||
* License Version 2 as published by the Free Software Foundation.
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* Library General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Library General Public
|
||||
* License along with this library; if not, write to the Free Software
|
||||
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
|
||||
* USA.
|
||||
*
|
||||
* Please report all bugs and problems to "retroshare@lunamutt.com".
|
||||
*
|
||||
*/
|
||||
|
||||
/*
|
||||
* Test for Multiplexor.....
|
||||
* As this is a key middle component, it is hard to test without other bits.
|
||||
* It relies on ftFileProvider/ftFileCreator/ftTransferModule...
|
||||
*
|
||||
* And has dummy ftDataSend and ftSearch.
|
||||
*
|
||||
*/
|
||||
|
||||
#ifdef WIN32
|
||||
#include "util/rswin.h"
|
||||
#endif
|
||||
|
||||
#include "ft/ftextralist.h"
|
||||
#include "ft/ftdatamultiplex.h"
|
||||
#include "ft/ftfilesearch.h"
|
||||
|
||||
#include "ftfileprovider.h"
|
||||
#include "ftfilecreator.h"
|
||||
#include "ftcontroller.h"
|
||||
#include "fttransfermodule.h"
|
||||
|
||||
#include "util/utest.h"
|
||||
|
||||
INITTEST()
|
||||
|
||||
|
||||
void do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list<std::string> &files);
|
||||
|
||||
|
||||
void usage(char *name)
|
||||
{
|
||||
std::cerr << "Usage: " << name << " [-p <period>] [-d <dperiod>] <path> [<path2> ... ] ";
|
||||
std::cerr << std::endl;
|
||||
}
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
int c;
|
||||
uint32_t period = 1;
|
||||
uint32_t dPeriod = 600; /* default 10 minutes */
|
||||
|
||||
std::list<std::string> fileList;
|
||||
|
||||
while(-1 != (c = getopt(argc, argv, "d:p:")))
|
||||
{
|
||||
switch (c)
|
||||
{
|
||||
case 'p':
|
||||
period = atoi(optarg);
|
||||
break;
|
||||
case 'd':
|
||||
dPeriod = atoi(optarg);
|
||||
break;
|
||||
default:
|
||||
usage(argv[0]);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (optind >= argc)
|
||||
{
|
||||
std::cerr << "Missing Files" << std::endl;
|
||||
usage(argv[0]);
|
||||
}
|
||||
|
||||
for(; optind < argc; optind++)
|
||||
{
|
||||
std::cerr << "Adding: " << argv[optind] << std::endl;
|
||||
fileList.push_back(std::string(argv[optind]));
|
||||
}
|
||||
|
||||
ftExtraList *eList = new ftExtraList();
|
||||
eList->start();
|
||||
|
||||
ftSearch *ftsd = new ftSearchDummy();
|
||||
ftFileSearch *ftfs = new ftFileSearch();
|
||||
|
||||
ftfs-> addSearchMode(ftsd, RS_FILE_HINTS_LOCAL);
|
||||
ftfs-> addSearchMode(eList, RS_FILE_HINTS_EXTRA);
|
||||
|
||||
ftDataSendPair *ftds1 = new ftDataSendPair(NULL);
|
||||
ftDataSendPair *ftds2 = new ftDataSendPair(NULL);
|
||||
|
||||
/* setup Actual Test bit */
|
||||
ftDataMultiplex *ftmplex1 = new ftDataMultiplex("ownId", ftds2, ftfs);
|
||||
ftDataMultiplex *ftmplex2 = new ftDataMultiplex("ownId", ftds1, ftfs);
|
||||
|
||||
ftds1->mDataRecv = ftmplex1;
|
||||
ftds2->mDataRecv = ftmplex2;
|
||||
|
||||
ftmplex1->start();
|
||||
ftmplex2->start();
|
||||
|
||||
/* Setup Search with some valid results */
|
||||
/* Request Data */
|
||||
|
||||
/* now work the thread */
|
||||
std::list<std::string>::iterator it;
|
||||
uint32_t flags = 0;
|
||||
for(it = fileList.begin(); it != fileList.end(); it++)
|
||||
{
|
||||
eList->hashExtraFile(*it, dPeriod, flags);
|
||||
}
|
||||
|
||||
if (fileList.size() < 1)
|
||||
{
|
||||
std::cerr << "come on, give us some files...";
|
||||
std::cerr << std::endl;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* now request files from ftDataMultiplex
|
||||
* by adding in a ftTransferModule!
|
||||
*/
|
||||
std::string filename = *(fileList.begin());
|
||||
|
||||
/* wait for file to hash */
|
||||
FileInfo info;
|
||||
while(!eList->hashExtraFileDone(filename, info))
|
||||
{
|
||||
std::cerr << "Waiting for file to hash";
|
||||
std::cerr << std::endl;
|
||||
sleep(1);
|
||||
}
|
||||
|
||||
std::string savename = "/tmp/" + info.fname;
|
||||
ftFileCreator *creator = new ftFileCreator(savename, info.size, info.hash, 0);
|
||||
ftController *controller = NULL;
|
||||
|
||||
ftTransferModule *transfer = new ftTransferModule(creator, ftmplex1, controller);
|
||||
|
||||
ftmplex1->addTransferModule(transfer, creator);
|
||||
|
||||
std::list<std::string> peerIds;
|
||||
peerIds.push_back("ownId2");
|
||||
|
||||
transfer->setFileSources(peerIds);
|
||||
transfer->setPeerState("ownId2", PQIPEER_IDLE, 1000);
|
||||
//transfer->resumeTransfer();
|
||||
|
||||
/* check file progress */
|
||||
while(1)
|
||||
{
|
||||
std::cerr << "File Transfer Status";
|
||||
std::cerr << std::endl;
|
||||
|
||||
std::cerr << "Transfered: " << creator->getRecvd();
|
||||
std::cerr << std::endl;
|
||||
|
||||
transfer->tick();
|
||||
sleep(1);
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user