Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-26 16:09:35 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

More Improvements to FileTransfer:

Browse Source 
* Added TlvShallowClear() to serialisers
 * Added RsQueueThread for periodic queue processes.
 * Completed ftDataMultiplex which replaces ftServer/ClientModules.
 * Added Server Queue / Thread to ftDataMultiplex.
 * Added ftdataplextest to exercise ftDataMultiplex.
 * Generalised ftFileSearch to handle an array of ftSearch classes.
 * Tweaked rsfiles.h #defines to match new ftFileSearch scheme.
 * Added Generic ftData Interfaces for Testing.
 * added ftDataSend/Recv Interfaces to ftServer + ftDataMultiplex respectively.
 * Completed much of ftServer (External Interface), but not yet done.
 * Extra debugging and small changes to ftExtraList
 * Integrated new ftTransferModule with minor interface changes.



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@660 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
drbob 2008-08-03 12:45:53 +00:00
parent d584516859
commit 5c6e558942
22 changed files with 1355 additions and 326 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
libretroshare/src/ft/Makefile
View File

@ -7,12 +7,13 @@ RS_TOP_DIR = ..
include $(RS_TOP_DIR)/scripts/config.mk
###############################################################
RSOBJ = ftfileprovider.o ftfilecreator.o ftextralist.o ftdatamultiplex.o
#ftservermodule.o
RSOBJ = ftdata.o ftfileprovider.o ftfilecreator.o ftextralist.o ftdatamultiplex.o ftfilesearch.o fttransfermodule.o
TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o
#ftserver.o
TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest
TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o
TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest
all: librs tests
@ -25,6 +26,9 @@ ftfileprovidertest : ftfileprovidertest.o
ftextralisttest : ftextralisttest.o
$(CC) $(CFLAGS) -o ftextralisttest ftextralisttest.o $(LIBS)
ftdataplextest : ftdataplextest.o
$(CC) $(CFLAGS) -o ftdataplextest ftdataplextest.o $(LIBS)
###############################################################
include $(RS_TOP_DIR)/scripts/rules.mk
###############################################################

8
libretroshare/src/ft/ftcontroller.h
View File

@ -38,6 +38,10 @@
*
*/
class ftFileCreator;
class ftTransferModule;
class ftFileProvider;
class ftController: public CacheTransfer, public RsThread, public pqiMonitor, public p3Config
{
public:
@ -96,8 +100,8 @@ virtual bool loadList(std::list<RsItem *> load);
RsMutex ctrlMutex;
std::list<FileDetails> incomingQueue;
std::map<std::string, FileDetails> mCompleted;
std::list<FileInfo> incomingQueue;
std::map<std::string, FileInfo> mCompleted;
std::map<std::string, ftTransferModule *> mTransfers;
std::map<std::string, ftFileCreator *> mFileCreators;

82
libretroshare/src/ft/ftdata.cc Normal file
View File

@ -0,0 +1,82 @@
/*
* libretroshare/src/ft: ftdata.cc
*
* File Transfer for RetroShare.
*
* Copyright 2008 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "ft/ftdata.h"
/******* Pair of Send/Recv (Only need to handle Send side) ******/
ftDataSendPair::ftDataSendPair(ftDataRecv *recv)
:mDataRecv(recv)
{
return;
}
/* Client Send */
bool ftDataSendPair::sendDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize)
{
return mDataRecv->recvDataRequest(peerId,hash,size,offset,chunksize);
}
/* Server Send */
bool ftDataSendPair::sendData(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{
return mDataRecv->recvData(peerId, hash,size,offset,chunksize,data);
}
/* Client Send */
bool ftDataSendDummy::sendDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize)
{
return true;
}
/* Server Send */
bool ftDataSendDummy::sendData(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{
return true;
}
/* Client Recv */
bool ftDataRecvDummy::recvData(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{
return true;
}
/* Server Recv */
bool ftDataRecvDummy::recvDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize)
{
return true;
}

64
libretroshare/src/ft/ftdata.h
View File

@ -47,11 +47,11 @@ virtual ~ftDataSend() { return; }
/* Client Send */
virtual bool sendDataRequest(std::string peerId, std::string hash,
uint64_t offset, uint32_t size) = 0;
uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
/* Server Send */
virtual bool sendData(std::string peerId, std::string hash,
uint64_t offset, uint32_t size, void *data) = 0;
virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data) = 0;
};
@ -66,12 +66,64 @@ class ftDataRecv
virtual ~ftDataRecv() { return; }
/* Client Recv */
virtual bool recvData(std::string peerId, std::string hash,
uint64_t offset, uint32_t size, void *data) = 0;
virtual bool recvData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data) = 0;
/* Server Recv */
virtual bool recvDataRequest(std::string peerId, std::string hash,
uint64_t offset, uint32_t size) = 0;
uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
};
/******* Pair of Send/Recv (Only need to handle Send side) ******/
class ftDataSendPair: public ftDataSend
{
public:
ftDataSendPair(ftDataRecv *recv);
virtual ~ftDataSendPair() { return; }
/* Client Send */
virtual bool sendDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize);
/* Server Send */
virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
ftDataRecv *mDataRecv;
};
class ftDataSendDummy: public ftDataSend
{
public:
virtual ~ftDataSendDummy() { return; }
/* Client Send */
virtual bool sendDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize);
/* Server Send */
virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
};
class ftDataRecvDummy: public ftDataRecv
{
public:
virtual ~ftDataRecvDummy() { return; }
/* Client Recv */
virtual bool recvData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
/* Server Recv */
virtual bool recvDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize);
};

140
libretroshare/src/ft/ftdatamultiplex.cc
View File

@ -35,6 +35,11 @@
#include "ft/ftfileprovider.h"
#include "ft/ftsearch.h"
/* For Thread Behaviour */
const uint32_t DMULTIPLEX_MIN = 10; /* 1ms sleep */
const uint32_t DMULTIPLEX_MAX = 1000; /* 1 sec sleep */
const double DMULTIPLEX_RELAX = 0.5; /* ??? */
ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
:mModule(module), mCreator(creator)
{
@ -44,15 +49,16 @@ ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
const uint32_t FT_DATA = 0x0001;
const uint32_t FT_DATA_REQ = 0x0002;
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t offset, uint32_t chunk, void *data)
:mType(type), mPeerId(peerId), mHash(hash),
ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
:mType(type), mPeerId(peerId), mHash(hash), mSize(size),
mOffset(offset), mChunk(chunk), mData(data)
{
return;
}
ftDataMultiplex::ftDataMultiplex(ftDataSend *server, ftSearch *search)
:mDataSend(server), mSearch(search)
:RsQueueThread(DMULTIPLEX_MIN, DMULTIPLEX_MAX, DMULTIPLEX_RELAX),
mDataSend(server), mSearch(search)
{
return;
}
@ -90,49 +96,135 @@ bool ftDataMultiplex::removeTransferModule(ftTransferModule *mod, ftFileCreator
/*************** SEND INTERFACE (calls ftDataSend) *******************/
/* Client Send */
bool ftDataMultiplex::sendDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size)
bool ftDataMultiplex::sendDataRequest(std::string peerId,
std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize)
{
return mDataSend->sendDataRequest(peerId, hash, offset, size);
return mDataSend->sendDataRequest(peerId,hash,size,offset,chunksize);
}
/* Server Send */
bool ftDataMultiplex::sendData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data)
bool ftDataMultiplex::sendData(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{
return mDataSend->sendData(peerId, hash, offset, size, data);
return mDataSend->sendData(peerId,hash,size,offset,chunksize,data);
}
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */
bool ftDataMultiplex::recvData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data)
bool ftDataMultiplex::recvData(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
mRequestQueue.push_back(
ftRequest(FT_DATA, peerId, hash, offset, size, data));
ftRequest(FT_DATA,peerId,hash,size,offset,chunksize,data));
return true;
}
/* Server Recv */
bool ftDataMultiplex::recvDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size)
bool ftDataMultiplex::recvDataRequest(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
{
/* Store in Queue */
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
mRequestQueue.push_back(
ftRequest(FT_DATA_REQ, peerId, hash, offset, size, NULL));
ftRequest(FT_DATA_REQ,peerId,hash,size,offset,chunksize,NULL));
return true;
}
/*********** BACKGROUND THREAD OPERATIONS ***********/
bool ftDataMultiplex::workQueued()
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
if (mRequestQueue.size() > 0)
{
return true;
}
if (mSearchQueue.size() > 0)
{
return true;
}
return false;
}
bool ftDataMultiplex::doWork()
{
bool doRequests = true;
/* Handle All the current Requests */
while(doRequests)
{
ftRequest req;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
if (mRequestQueue.size() == 0)
{
doRequests = false;
continue;
}
req = mRequestQueue.front();
mRequestQueue.pop_front();
}
/* MUTEX FREE */
switch(req.mType)
{
case FT_DATA:
handleRecvData(req.mPeerId, req.mHash, req.mSize,
req.mOffset, req.mChunk, req.mData);
break;
case FT_DATA_REQ:
handleRecvDataRequest(req.mPeerId, req.mHash,
req.mSize, req.mOffset, req.mChunk);
break;
default:
break;
}
}
/* Only Handle One Search Per Period....
* Lower Priority
*/
ftRequest req;
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
if (mSearchQueue.size() == 0)
{
/* Finished */
return true;
}
req = mSearchQueue.front();
mSearchQueue.pop_front();
}
handleSearchRequest(req.mPeerId, req.mHash, req.mSize,
req.mOffset, req.mChunk);
return true;
}
bool ftDataMultiplex::handleRecvData(std::string peerId,
std::string hash, uint64_t offset, uint32_t size, void *data)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data)
{
RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
std::map<std::string, ftClient>::iterator it;
@ -142,7 +234,7 @@ bool ftDataMultiplex::handleRecvData(std::string peerId,
return false;
}
(it->second).mModule->recvFileData(peerId, offset, size, data);
(it->second).mModule->recvFileData(peerId, offset, chunksize, data);
return true;
}
@ -150,7 +242,8 @@ bool ftDataMultiplex::handleRecvData(std::string peerId,
/* called by ftTransferModule */
bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
std::string hash, uint64_t offset, uint32_t size)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
{
/**** Find Files *****/
@ -159,7 +252,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
if (mClients.end() != (cit = mClients.find(hash)))
{
locked_handleServerRequest((cit->second).mCreator,
peerId, hash, offset, size);
peerId, hash, size, offset, chunksize);
return true;
}
@ -167,26 +260,28 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId,
if (mServers.end() != (sit = mServers.find(hash)))
{
locked_handleServerRequest(sit->second,
peerId, hash, offset, size);
peerId, hash, size, offset, chunksize);
return true;
}
/* Add to Search Queue */
mSearchQueue.push_back(
ftRequest(FT_DATA_REQ, peerId, hash, offset, size, NULL));
ftRequest(FT_DATA_REQ, peerId, hash,
size, offset, chunksize, NULL));
return true;
}
bool ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider,
std::string peerId, std::string hash, uint64_t offset, uint32_t size)
std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
{
void *data = malloc(size);
if (provider->getFileData(offset, size, data))
if (provider->getFileData(offset, chunksize, data))
{
/* send data out */
sendData(peerId, hash, offset, size, data);
sendData(peerId, hash, size, offset, chunksize, data);
return true;
}
return false;
@ -194,7 +289,8 @@ bool ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider,
bool ftDataMultiplex::handleSearchRequest(std::string peerId,
std::string hash, uint64_t offset, uint32_t size)
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize)
{
{
@ -235,7 +331,7 @@ bool ftDataMultiplex::handleSearchRequest(std::string peerId,
/* handle request finally */
locked_handleServerRequest(provider,
peerId, hash, offset, size);
peerId, hash, size, offset, chunksize);
return true;
}
return false;

36
libretroshare/src/ft/ftdatamultiplex.h
View File

@ -61,11 +61,15 @@ class ftRequest
{
public:
ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t offset, uint32_t chunk, void *data);
ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data);
ftRequest()
:mType(0), mSize(0), mOffset(0), mChunk(0), mData(NULL) { return; }
uint32_t mType;
std::string mPeerId;
std::string mHash;
uint64_t mSize;
uint64_t mOffset;
uint32_t mChunk;
void *mData;
@ -73,7 +77,7 @@ class ftRequest
class ftDataMultiplex: public ftDataRecv
class ftDataMultiplex: public ftDataRecv, public RsQueueThread
{
public:
@ -89,36 +93,48 @@ bool removeTransferModule(ftTransferModule *mod, ftFileCreator *f);
/*************** SEND INTERFACE (calls ftDataSend) *******************/
/* Client Send */
bool sendDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size);
bool sendDataRequest(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize);
/* Server Send */
bool sendData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data);
bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
/*************** RECV INTERFACE (provides ftDataRecv) ****************/
/* Client Recv */
virtual bool recvData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data);
virtual bool recvData(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
/* Server Recv */
virtual bool recvDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size);
virtual bool recvDataRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
protected:
/* Overloaded from RsQueueThread */
virtual bool workQueued();
virtual bool doWork();
private:
/* Handling Job Queues */
bool handleRecvData(std::string peerId,
std::string hash, uint64_t offset, uint32_t size, void *data);
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
bool handleRecvDataRequest(std::string peerId,
std::string hash, uint64_t offset, uint32_t size);
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize);
bool handleSearchRequest(std::string peerId,
std::string hash, uint64_t offset, uint32_t size);
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize);
/* We end up doing the actual server job here */
bool locked_handleServerRequest(ftFileProvider *provider,
std::string peerId, std::string hash, uint64_t offset, uint32_t size);
std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize);
RsMutex dataMtx;

174
libretroshare/src/ft/ftdataplextest.cc Normal file
View File

@ -0,0 +1,174 @@
/*
* libretroshare/src/ft: ftextralisttest.cc
*
* File Transfer for RetroShare.
*
* Copyright 2008 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
/*
* Test for Multiplexor.....
* As this is a key middle component, it is hard to test without other bits.
* It relies on ftFileProvider/ftFileCreator/ftTransferModule...
*
* And has dummy ftDataSend and ftSearch.
*
*/
#include "ft/ftextralist.h"
#include "ft/ftdatamultiplex.h"
#include "ft/ftfilesearch.h"
void do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list<std::string> &files);
void usage(char *name)
{
std::cerr << "Usage: " << name << " [-p <period>] [-d <dperiod>] <path> [<path2> ... ] ";
std::cerr << std::endl;
}
int main(int argc, char **argv)
{
int c;
uint32_t period = 1;
uint32_t dPeriod = 600; /* default 10 minutes */
std::list<std::string> fileList;
while(-1 != (c = getopt(argc, argv, "d:p:")))
{
switch (c)
{
case 'p':
period = atoi(optarg);
break;
case 'd':
dPeriod = atoi(optarg);
break;
default:
usage(argv[0]);
break;
}
}
if (optind >= argc)
{
std::cerr << "Missing Files" << std::endl;
usage(argv[0]);
}
for(; optind < argc; optind++)
{
std::cerr << "Adding: " << argv[optind] << std::endl;
fileList.push_back(std::string(argv[optind]));
}
ftExtraList *eList = new ftExtraList();
eList->start();
ftSearch *ftsd = new ftSearchDummy();
ftFileSearch *ftfs = new ftFileSearch();
ftfs-> addSearchMode(ftsd, RS_FILE_HINTS_LOCAL);
ftfs-> addSearchMode(eList, RS_FILE_HINTS_EXTRA);
ftDataSend *ftds = new ftDataSendDummy();
/* setup Actual Test bit */
ftDataMultiplex *ftmplex = new ftDataMultiplex(ftds, ftfs);
ftmplex->start();
/* Setup Search with some valid results */
/* Request Data */
/* now work the thread */
std::list<std::string>::iterator it;
uint32_t flags = 0;
for(it = fileList.begin(); it != fileList.end(); it++)
{
eList->hashExtraFile(*it, dPeriod, flags);
}
/* now request files from ftDataMultiplex */
/* just request random data packets first */
do_random_server_test(ftmplex, eList, fileList);
}
uint32_t do_random_server_iteration(ftDataMultiplex *mplex, ftExtraList *eList, std::list<std::string> &files)
{
std::cerr << "do_random_server_iteration()";
std::cerr << std::endl;
std::list<std::string>::iterator it;
uint32_t i = 0;
for(it = files.begin(); it != files.end(); it++)
{
FileInfo info;
if (eList->hashExtraFileDone(*it, info))
{
std::cerr << "Hash Done for: " << *it;
std::cerr << std::endl;
std::cerr << info << std::endl;
std::cerr << "Requesting Data Packet";
std::cerr << std::endl;
/* Server Recv */
uint64_t offset = 10000;
uint32_t chunk = 20000;
mplex->recvDataRequest("Peer", info.hash, info.size, offset, chunk);
i++;
}
else
{
std::cerr << "do_random_server_iteration() Hash Not Done for: " << *it;
std::cerr << std::endl;
}
}
return i;
}
void do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list<std::string> &files)
{
std::cerr << "do_random_server_test()";
std::cerr << std::endl;
uint32_t size = files.size();
while(size > do_random_server_iteration(mplex, eList, files))
{
std::cerr << "do_random_server_test() sleep";
std::cerr << std::endl;
sleep(10);
}
}

88
libretroshare/src/ft/ftextralist.cc
View File

@ -26,6 +26,8 @@
#include "ft/ftextralist.h"
#include "util/rsdir.h"
#define DEBUG_ELIST 1
ftExtraList::ftExtraList()
:p3Config(CONFIG_FT_EXTRA_LIST)
{
@ -41,6 +43,11 @@ void ftExtraList::run()
while (1)
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::run() Iteration";
std::cerr << std::endl;
#endif
now = time(NULL);
{
@ -76,15 +83,29 @@ void ftExtraList::run()
void ftExtraList::hashAFile()
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::hashAFile()";
std::cerr << std::endl;
#endif
/* extract entry from the queue */
FileDetails details;
{
RsStackMutex stack(extMutex);
if (mToHash.size() == 0)
return;
details = mToHash.front();
mToHash.pop_front();
}
#ifdef DEBUG_ELIST
std::cerr << "Hashing: " << details.info.path;
std::cerr << std::endl;
#endif
/* hash it! */
std::string name, hash;
uint64_t size;
@ -110,6 +131,16 @@ void ftExtraList::hashAFile()
bool ftExtraList::addExtraFile(std::string path, std::string hash,
uint64_t size, uint32_t period, uint32_t flags)
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::addExtraFile() path: " << path;
std::cerr << " hash: " << hash;
std::cerr << " size: " << size;
std::cerr << " period: " << period;
std::cerr << " flags: " << flags;
std::cerr << std::endl;
#endif
RsStackMutex stack(extMutex);
FileDetails details;
@ -126,11 +157,42 @@ bool ftExtraList::addExtraFile(std::string path, std::string hash,
/* stick it in the available queue */
mFiles[details.info.hash] = details;
return true;
}
bool ftExtraList::removeExtraFile(std::string hash, uint32_t flags)
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::removeExtraFile()";
std::cerr << " hash: " << hash;
std::cerr << " flags: " << flags;
std::cerr << std::endl;
#endif
RsStackMutex stack(extMutex);
std::map<std::string, FileDetails>::iterator it;
it = mFiles.find(hash);
if (it == mFiles.end())
{
return false;
}
mFiles.erase(it);
return true;
}
bool ftExtraList::cleanupOldFiles()
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::cleanupOldFiles()";
std::cerr << std::endl;
#endif
RsStackMutex stack(extMutex);
time_t now = time(NULL);
@ -159,6 +221,7 @@ bool ftExtraList::cleanupOldFiles()
}
}
}
return true;
}
@ -169,6 +232,14 @@ bool ftExtraList::cleanupOldFiles()
bool ftExtraList::hashExtraFile(std::string path, uint32_t period, uint32_t flags)
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::hashExtraFile() path: " << path;
std::cerr << " period: " << period;
std::cerr << " flags: " << flags;
std::cerr << std::endl;
#endif
/* add into queue */
RsStackMutex stack(extMutex);
@ -180,6 +251,11 @@ bool ftExtraList::hashExtraFile(std::string path, uint32_t period, uint32_t fla
bool ftExtraList::hashExtraFileDone(std::string path, FileInfo &info)
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::hashExtraFileDone()";
std::cerr << std::endl;
#endif
std::string hash;
{
/* Find in the path->hash map */
@ -192,19 +268,25 @@ bool ftExtraList::hashExtraFileDone(std::string path, FileInfo &info)
}
hash = it->second;
}
return searchExtraFiles(hash, info);
return search(hash, 0, 0, info);
}
/***
* Search Function - used by File Transfer
*
**/
bool ftExtraList::searchExtraFiles(std::string hash, FileInfo &info)
bool ftExtraList::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
{
#ifdef DEBUG_ELIST
std::cerr << "ftExtraList::search()";
std::cerr << std::endl;
#endif
RsStackMutex stack(extMutex);
/* find hash */
std::map<std::string, FileDetails>::iterator fit;
std::map<std::string, FileDetails>::const_iterator fit;
if (mFiles.end() == (fit = mFiles.find(hash)))
{
return false;

10
libretroshare/src/ft/ftextralist.h
View File

@ -57,6 +57,7 @@
#include <map>
#include <string>
#include "ft/ftsearch.h"
#include "util/rsthreads.h"
#include "rsiface/rsfiles.h"
#include "pqi/p3cfgmgr.h"
@ -108,7 +109,7 @@ const uint32_t CONFIG_FT_EXTRA_LIST = 1;
const uint32_t CLEANUP_PERIOD = 600; /* 10 minutes */
class ftExtraList: public RsThread, public p3Config
class ftExtraList: public RsThread, public p3Config, public ftSearch
{
public:
@ -122,6 +123,8 @@ class ftExtraList: public RsThread, public p3Config
bool addExtraFile(std::string path, std::string hash,
uint64_t size, uint32_t period, uint32_t flags);
bool removeExtraFile(std::string hash, uint32_t flags);
/***
* Hash file, and add to the files,
* file is removed after period.
@ -132,9 +135,10 @@ bool hashExtraFileDone(std::string path, FileInfo &info);
/***
* Search Function - used by File Transfer
* implementation of ftSearch.
*
**/
bool searchExtraFiles(std::string hash, FileInfo &info);
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
/***
* Thread Main Loop
@ -156,7 +160,7 @@ virtual bool loadList(std::list<RsItem *> load);
void hashAFile();
bool cleanupOldFiles();
RsMutex extMutex;
mutable RsMutex extMutex;
std::list<FileDetails> mToHash;

2
libretroshare/src/ft/ftextralisttest.cc
View File

@ -142,7 +142,7 @@ void displayExtraListDetails(ftExtraList *eList, std::list<std::string> toHash,
for(it = hashed.begin(); it != hashed.end(); it++)
{
FileInfo info;
if (eList->searchExtraFiles(*it, info))
if (eList->search(*it, 0, 0, info))
{
std::cerr << "displayExtraListDetails() Found Hash: " << *it;
std::cerr << std::endl;

10
libretroshare/src/ft/ftfileprovider.cc
View File

@ -11,6 +11,16 @@ ftFileProvider::~ftFileProvider(){
}
}
std::string ftFileProvider::getHash()
{
return hash;
}
uint64_t ftFileProvider::getFileSize()
{
return total_size;
}
bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *data)
{
RsStackMutex stack(ftPMutex); /********** STACK LOCKED MTX ******/

165
libretroshare/src/ft/ftfilesearch.cc
View File

@ -23,57 +23,142 @@
*
*/
#ifndef FT_FILE_SEARCH_HEADERd
#define FT_FILE_SEARCH_HEADER
/*
* ftFileSearch
*
* This is actually implements the ftSearch Interface.
*
*/
#include "ft/ftfilesearch.h"
#include "dbase/cachestrapper.h"
#include "dbase/fimonitor.h"
#include "dbase/fistore.h"
bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info)
const uint32_t MAX_SEARCHS = 24; /* lower 24 bits of hint */
ftFileSearch::ftFileSearch()
:mSearchs(MAX_SEARCHS)
{
#if 0
/* actual search depends on the hints */
if (hintflags | CACHE)
uint32_t i;
for(i = 0; i < MAX_SEARCHS; i++)
{
mCacheStrapper->..
mSearchs[i] = NULL;
}
}
if (hintflags | LOCAL)
{
bool ftFileSearch::addSearchMode(ftSearch *search, uint32_t hintflags)
{
hintflags &= 0x00ffffff;
}
if (hintflags | EXTRA)
{
}
if (hintflags | REMOTE)
{
}
private:
CacheStrapper *mCacheStrapper;
ftExtraList *mExtraList;
FileIndexMonitor *mFileMonitor;
FileIndexStore *mFileStore;
#ifndef DEBUG_SEARCH
std::cerr << "ftSearchDummy::addSearchMode() : " << hintflags;
std::cerr << std::endl;
#endif
uint32_t i;
for (i = 0; i < MAX_SEARCHS; i++)
{
uint32_t hints = hintflags >> i;
if (hints & 0x0001)
{
/* has the flag */
mSearchs[i] = search;
#ifndef DEBUG_SEARCH
std::cerr << "ftSearchDummy::addSearchMode() to slot ";
std::cerr << i;
std::cerr << std::endl;
#endif
return true;
}
}
#ifndef DEBUG_SEARCH
std::cerr << "ftSearchDummy::addSearchMode() Failed";
std::cerr << std::endl;
#endif
return false;
}
bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
{
uint32_t hints, i;
#ifndef DEBUG_SEARCH
std::cerr << "ftFileSearch::search(" << hash << ", " << size;
std::cerr << ", " << hintflags << ");";
std::cerr << std::endl;
#endif
for (i = 0; i < MAX_SEARCHS; i++)
{
hints = hintflags >> i;
if (hints & 0x0001)
{
/* has the flag */
ftSearch *search = mSearchs[i];
if (search)
{
#ifndef DEBUG_SEARCH
std::cerr << "ftFileSearch::search() SLOT: ";
std::cerr << i;
std::cerr << std::endl;
#endif
if (search->search(hash, size, hintflags, info))
{
return true;
}
}
}
}
/* if we haven't found it by now! - check if SPEC_ONLY flag is set */
if (hintflags & RS_FILE_HINTS_SPEC_ONLY)
{
#ifndef DEBUG_SEARCH
std::cerr << "ftFileSearch::search() SPEC_ONLY: Failed";
std::cerr << std::endl;
#endif
return false;
}
#ifndef DEBUG_SEARCH
std::cerr << "ftSearchDummy::search() Searching Others:";
std::cerr << std::endl;
#endif
/* if we don't have the SPEC_ONLY flag,
* we check through all the others
*/
for (i = 0; i < MAX_SEARCHS; i++)
{
hints = hintflags >> i;
if (hints & 0x0001)
{
continue;
}
/* has the flag */
ftSearch *search = mSearchs[i];
if (search)
{
#ifndef DEBUG_SEARCH
std::cerr << "ftFileSearch::search() SLOT: " << i;
std::cerr << std::endl;
#endif
if (search->search(hash, size, hintflags, info))
{
return true;
}
}
}
/* found nothing */
return false;
}
bool ftSearchDummy::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
{
#ifndef DEBUG_SEARCH
std::cerr << "ftSearchDummy::search(" << hash << ", " << size;
std::cerr << ", " << hintflags << ");";
std::cerr << std::endl;
#endif
return false;
}

25
libretroshare/src/ft/ftfilesearch.h
View File

@ -29,36 +29,31 @@
/*
* ftFileSearch
*
* This is actually implements the ftSearch Interface.
* This is implements an array of ftSearch Interfaces.
*
*/
#include "ft/ftsearch.h"
#include <vector>
class CacheStrapper;
class ftExtraList;
class FileIndexMonitor;
class FileIndexStore;
#include "ft/ftsearch.h"
class ftFileSearch: public ftSearch
{
public:
ftFileSearch(CacheStrapper *c, ftExtraList *x, FileIndexMonitor *m, FileIndexStore *s)
:mCacheStrapper(c), mExtraList(x), mFileMonitor(m), mFileStore(s) { return; }
ftFileSearch();
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info);
bool addSearchMode(ftSearch *search, uint32_t hintflags);
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
private:
CacheStrapper *mCacheStrapper;
ftExtraList *mExtraList;
FileIndexMonitor *mFileMonitor;
FileIndexStore *mFileStore;
// should have a mutex to protect vector....
// but not really necessary as it is const most of the time.
std::vector<ftSearch *> mSearchs;
};
#endif

12
libretroshare/src/ft/ftsearch.h
View File

@ -43,20 +43,18 @@ class ftSearch
ftSearch() { return; }
virtual ~ftSearch() { return; }
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) = 0;
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const = 0;
};
class ftSearchDummy
class ftSearchDummy: public ftSearch
{
public:
ftSearchDummy() { return; }
virtual ~ftSearchDummy() { return; }
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info)
{
return false;
}
virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
};
#endif

453
libretroshare/src/ft/ftserver.cc
View File

@ -23,16 +23,56 @@
*
*/
#include "util/rsdebug.h"
const int ftserverzone = 29539;
#include "ft/ftserver.h"
#include "ft/ftextralist.h"
#include "ft/ftfilesearch.h"
#include "ft/ftcontroller.h"
#include "ft/ftdatamultiplex.h"
#include "dbase/cachestrapper.h"
#include "dbase/fimonitor.h"
#include "dbase/fistore.h"
#include "pqi/pqi.h"
#include "pqi/p3connmgr.h"
#include <iostream>
#include <sstream>
/* Setup */
ftServer::ftServer(CacheStrapper *cStrapper, p3ConnectMgr *connMgr)
:mCacheStrapper(cStrapper), mConnMgr(connMgr)
{
}
void ftServer::setConfigDirectory(std::string path)
{
mConfigPath = path;
}
void ftServer::setPQInterface(PQInterface *pqi)
{
}
/* Control Interface */
/* add Config Items (Extra, Controller) */
void ftServer::addConfigComponents(p3ConfigMgr *mgr)
{
/* NOT SURE ABOUT THIS ONE */
}
/* Final Setup (once everything is assigned) */
ftServer::SetupFtServer()
void ftServer::SetupFtServer()
{
/* make Controller */
mFtController = new ftController();
mFtController = new ftController(config_dir);
NotifyBase *cb = getNotify();
/* setup FiStore/Monitor */
@ -49,43 +89,42 @@ ftServer::SetupFtServer()
/* extras List */
mFtExtra = new ftExtraList();
mFtSearch = new ftFileSearch(mCacheStrapper, mFtExtra, mFiMon, mFiStore);
mFtSearch = new ftFileSearch();
//mFtSearch->addSearchMode(mCacheStrapper, RS_FILE_HINTS_CACHE);
mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA);
//mFtSearch->addSearchMode(mFiMon, RS_FILE_HINTS_LOCAL);
//mFtSearch->addSearchMode(mFiStore, RS_FILE_HINTS_REMOTE);
mFtController -> setFtSearch(mFtSearch);
ftFiler -> setSaveBasePath(save_dir);
mFtController -> setSaveBasePath(save_dir);
mFtDataplex = ftDataMultiplex(this, mFtSearch);
return;
}
/* Assign important variables */
void setConfigDirectory(std::string path);
void setPQInterface(PQInterface *pqi);
/* Final Setup (once everything is assigned) */
void SetupFtServer();
/* add Config Items (Extra, Controller) */
void addConfigComponents(p3ConfigMgr *mgr);
void ftServer::setConfigDirectory(std::string path)
{
mConfigPath = path;
}
void ftServer::setPQInterface(PQInterface *pqi)
/* Control Interface */
};
void ftServer::StartupThreads()
{
/* start up Controller thread */
/* start up order - important for dependencies */
/* self contained threads */
/* startup ExtraList Thread */
mFtExtra->start();
/* startup Monitor Thread */
/* startup the FileMonitor (after cache load) */
mFiMon->setPeriod(600); /* 10 minutes */
/* start it up */
//mFiMon->setSharedDirectories(dbase_dirs);
mFiMon->start();
/* start own thread */
//start();
/* Controller thread */
mFtController->start();
}
CacheStrapper *ftServer::getCacheStrapper()
@ -98,6 +137,7 @@ CacheTransfer *ftServer::getCacheTransfer()
return mFtController;
}
/***************************************************************/
/********************** RsFiles Interface **********************/
/***************************************************************/
@ -113,23 +153,23 @@ bool ftServer::FileRequest(std::string fname, std::string hash,
return mFtController->FileRequest(fname, hash, size, dest, flags);
}
bool ftServer::FileCancel(std::string hash);
bool ftServer::FileCancel(std::string hash)
{
return mFtController->FileCancel(hash);
}
bool ftServer::FileControl(std::string hash, uint32_t flags);
bool ftServer::FileControl(std::string hash, uint32_t flags)
{
return mFtController->FileControl(hash, flags);
}
bool ftServer::FileClearCompleted();
bool ftServer::FileClearCompleted()
{
return mFtController->FileClearCompleted();
}
/* get Details of File Transfers */
bool ftServer::FileDownloads(std::list<std::string> &hashs);
bool ftServer::FileDownloads(std::list<std::string> &hashs)
{
return mFtController->FileDownloads(hashs);
}
@ -137,7 +177,7 @@ bool ftServer::FileDownloads(std::list<std::string> &hashs);
/* Directory Handling */
void ftServer::setDownloadDirectory(std::string path)
{
return mFtController->setDownloadDirectory(path);
mFtController->setDownloadDirectory(path);
}
std::string ftServer::getDownloadDirectory()
@ -145,12 +185,12 @@ std::string ftServer::getDownloadDirectory()
return mFtController->getDownloadDirectory();
}
void ftServer::setPartialsDirectory(std::string path);
void ftServer::setPartialsDirectory(std::string path)
{
return mFtController->setPartialsDirectory(path);
mFtController->setPartialsDirectory(path);
}
void ftServer::getPartialsDirectory()
std::string ftServer::getPartialsDirectory()
{
return mFtController->getPartialsDirectory();
}
@ -160,21 +200,21 @@ void ftServer::getPartialsDirectory()
/************************* Other Access ************************/
/***************************************************************/
bool ftServer::FileUploads(std::list<std::string> &hashs);
bool ftServer::FileUploads(std::list<std::string> &hashs)
{
return mFtUploader->FileUploads(hashes);
return mFtDataplex->FileUploads(hashes);
}
bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info);
bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info)
{
bool found = false;
if (hintflags | DOWNLOADING)
if (hintflags | RS_FILE_HINTS_DOWNLOAD)
{
found = mFtController->FileDetails(hash, info);
}
else if (hintflags | UPLOADING)
else if (hintflags | RS_FILE_HINTS_UPLOAD)
{
found = mFtUploader->FileDetails(hash, info);
found = mFtDataplex->FileDetails(hash, info);
}
if (!found)
@ -194,17 +234,17 @@ bool ftServer::ExtraFileAdd(std::string fname, std::string hash, uint32_t size,
return mFtExtra->addExtraFile(fname, hash, size, period, flags);
}
bool ftServer::ExtraFileRemove(std::string hash, uin32_t flags);
bool ftServer::ExtraFileRemove(std::string hash, uint32_t flags)
{
return mFtExtra->removeExtraFile(hash, flags);
}
bool ftServer::ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags);
bool ftServer::ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags)
{
return mFtExtra->hashExtraFile(localpath, period, flags);
}
bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info);
bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info)
{
return mFtExtra->hashExtraFileDone(localpath, info);
}
@ -213,12 +253,12 @@ bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info);
/******************** Directory Listing ************************/
/***************************************************************/
int ftServer::RequestDirDetails(std::string uid, std::string path, DirDetails &details);
int ftServer::RequestDirDetails(std::string uid, std::string path, DirDetails &details)
{
return mFiStore->RequestDirDetails(uid, path, details);
}
int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags);
int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags)
{
return mFiStore->RequestDirDetails(ref, details, flags);
}
@ -228,12 +268,12 @@ int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags);
/***************************************************************/
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results);
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results)
{
return mFiStore->SearchKeywords(keywords, results);
}
int ftServer::SearchBoolExp(Expression * exp, std::list<FileDetail> &results);
int ftServer::SearchBoolExp(Expression * exp, std::list<FileDetail> &results)
{
return mFiStore->searchBoolExp(exp, results);
}
@ -256,22 +296,22 @@ void ftServer::ForceDirectoryCheck()
bool ftServer::InDirectoryCheck()
{
return mFtMon->inDirectoryCheck();
return mFiMon->inDirectoryCheck();
}
bool ftServer::getSharedDirectories(std::list<std::string> &dirs)
{
return mFtMon->getSharedDirectories(dirs);
return mFiMon->getSharedDirectories(dirs);
}
bool ftServer::addSharedDirectory(std::string dir)
{
return mFtMon->addSharedDirectory(dir);
return mFiMon->addSharedDirectory(dir);
}
bool ftServer::removeSharedDirectory(std::string dir)
{
return mFtMon->removeSharedDirectory(dir);
return mFiMon->removeSharedDirectory(dir);
}
@ -286,16 +326,309 @@ bool ftServer::removeSharedDirectory(std::string dir)
protected:
/* Key Functions to be overloaded for Full Configuration */
virtual RsSerialiser *setupSerialiser();
virtual std::list<RsItem *> saveList(bool &cleanup);
virtual bool loadList(std::list<RsItem *> load);
RsSerialiser *ftServer::setupSerialiser()
{
return NULL;
}
private:
bool loadConfigMap(std::map<std::string, std::string> &configMap);
/******************* p3 Config Overload ************************/
std::list<RsItem *> ftServer::saveList(bool &cleanup)
{
std::list<RsItem *> list;
return list;
}
};
bool ftServer::loadList(std::list<RsItem *> load)
{
return true;
}
bool ftServer::loadConfigMap(std::map<std::string, std::string> &configMap)
{
return true;
}
/***************************************************************/
/********************** Data Flow **********************/
/***************************************************************/
/* Client Send */
bool ftServer::sendDataRequest(std::string peerId, std::string hash,
uint64_t size, uint64_t offset, uint32_t chunksize)
{
/* create a packet */
/* push to networking part */
RsFileRequest *rfi = new RsFileRequest();
/* id */
rfi->PeerId(peerId);
/* file info */
rfi->file.filesize = size;
rfi->file.hash = hash; /* ftr->hash; */
/* offsets */
rfi->fileoffset = offset; /* ftr->offset; */
rfi->chunksize = chunksize; /* ftr->chunk; */
mP3iface->SendFileRequest(rfi);
}
const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */
/* Server Send */
bool ftServer::sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t baseoffset, uint32_t chunksize, void *data)
{
/* create a packet */
/* push to networking part */
uint32_t tosend = chunksize;
uint64_t offset = 0;
uint32_t chunk;
while(tosend > 0)
{
/* workout size */
chunk = MAX_FT_CHUNK;
if (chunk > tosend)
{
chunk = tosend;
}
/******** New Serialiser Type *******/
RsFileData *rfd = new RsFileData();
/* set id */
rfd->PeerId(peerId);
/* file info */
rfd->fd.file.filesize = size;
rfd->fd.file.hash = hash;
rfd->fd.file.name = ""; /* blank other data */
rfd->fd.file.path = "";
rfd->fd.file.pop = 0;
rfd->fd.file.age = 0;
rfd->fd.file_offset = baseoffset + offset;
/* file data */
rfd->fd.binData.setBinData(
&(((uint8_t *) data)[offset]), chunk);
mP3iface->SendFileData(rfd);
offset += chunk;
tosend -= chunk;
}
/* clean up data */
free(data);
}
int ftServer::tick()
{
rslog(RSL_DEBUG_BASIC, ftserverzone,
"filedexserver::tick()");
/* the new Cache Hack() */
FileStoreTick();
if (mP3iface == NULL)
{
std::ostringstream out;
rslog(RSL_DEBUG_BASIC, ftserverzone,
"filedexserver::tick() Invalid Interface()");
return 1;
}
int moreToTick = 0;
if (0 < mP3iface -> tick())
{
moreToTick = 1;
#ifdef DEBUG_TICK
std::cerr << "filedexserver::tick() moreToTick from mP3iface" << std::endl;
#endif
}
if (0 < handleInputQueues())
{
moreToTick = 1;
#ifdef DEBUG_TICK
std::cerr << "filedexserver::tick() moreToTick from InputQueues" << std::endl;
#endif
}
return moreToTick;
}
// This function needs to be divided up.
bool ftServer::handleInputQueues()
{
handleCacheData();
handleFileData();
}
bool ftServer::handleCacheData()
{
// get all the incoming results.. and print to the screen.
RsCacheRequest *cr;
RsCacheItem *ci;
// Loop through Search Results.
int i = 0;
int i_init = 0;
//std::cerr << "filedexserver::handleInputQueues()" << std::endl;
while((ci = mP3iface -> GetSearchResult()) != NULL)
{
//std::cerr << "filedexserver::handleInputQueues() Recvd SearchResult (CacheResponse!)" << std::endl;
std::ostringstream out;
if (i++ == i_init)
{
out << "Recieved Search Results:" << std::endl;
}
ci -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
/* these go to the CacheStrapper! */
CacheData data;
data.cid = CacheId(ci->cacheType, ci->cacheSubId);
data.hash = ci->file.hash;
data.size = ci->file.filesize;
data.name = ci->file.name;
data.path = ci->file.path;
data.pid = ci->PeerId();
data.pname = mAuthMgr->getName(ci->PeerId());
mCacheStrapper->recvCacheResponse(data, time(NULL));
delete ci;
}
// now requested Searches.
i_init = i;
while((cr = mP3iface -> RequestedSearch()) != NULL)
{
/* just delete these */
std::ostringstream out;
out << "Requested Search:" << std::endl;
cr -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
delete cr;
}
// Now handle it replacement (pushed cache results)
{
std::list<std::pair<RsPeerId, CacheData> > cacheUpdates;
std::list<std::pair<RsPeerId, CacheData> >::iterator it;
mCacheStrapper->getCacheUpdates(cacheUpdates);
for(it = cacheUpdates.begin(); it != cacheUpdates.end(); it++)
{
/* construct reply */
RsCacheItem *ci = new RsCacheItem();
/* id from incoming */
ci -> PeerId(it->first);
ci -> file.hash = (it->second).hash;
ci -> file.name = (it->second).name;
ci -> file.path = ""; // (it->second).path;
ci -> file.filesize = (it->second).size;
ci -> cacheType = (it->second).cid.type;
ci -> cacheSubId = (it->second).cid.subid;
#ifdef SERVER_DEBUG
std::ostringstream out2;
out2 << "Outgoing CacheStrapper Update -> RsCacheItem:" << std::endl;
ci -> print(out2);
std::cerr << out2.str() << std::endl;
#endif
//rslog(RSL_DEBUG_BASIC, ftserverzone, out2.str());
mP3iface -> SendSearchResult(ci);
}
}
}
bool ftServer::handleFileData()
{
// now File Input.
RsFileRequest *fr;
RsFileData *fd;
int i_init = 0;
int i = 0;
i_init = i;
while((fr = mP3iface -> GetFileRequest()) != NULL )
{
#ifdef SERVER_DEBUG
std::cerr << "filedexserver::handleInputQueues() Recvd ftFiler Request" << std::endl;
std::ostringstream out;
if (i == i_init)
{
out << "Incoming(Net) File Item:" << std::endl;
}
fr -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
#endif
i++; /* count */
mFtDataplex->recvDataRequest(fr->PeerId(),
fr->file.hash, fr->file.filesize,
fr->fileoffset, fr->chunksize);
FileInfo(ffr);
delete fr;
}
// now File Data.
i_init = i;
while((fd = mP3iface -> GetFileData()) != NULL )
{
#ifdef SERVER_DEBUG
//std::cerr << "filedexserver::handleInputQueues() Recvd ftFiler Data" << std::endl;
std::ostringstream out;
if (i == i_init)
{
out << "Incoming(Net) File Data:" << std::endl;
}
fd -> print(out);
rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
#endif
i++; /* count */
/* incoming data */
mFtDataplex->recvData(fd->PeerId(),
fd->fd.file.hash, fd->fd.file.filesize,
fd->fd.file_offset,
fd->fd.binData.bin_len,
fd->fd.binData.bin_data);
/* we've stolen the data part -> so blank before delete
*/
fd->fd.TlvShallowClear();
delete fd;
}
if (i > 0)
{
return 1;
}
return 0;
}
/**********************************
**********************************
**********************************
*********************************/

81
libretroshare/src/ft/ftserver.h
View File

@ -43,9 +43,11 @@
#include <list>
#include <iostream>
#include "ft/ftdata.h"
#include "rsiface/rsfiles.h"
#include "dbase/cachestrapper.h"
#include "pqi/pqi.h"
#include "pqi/p3cfgmgr.h"
@ -58,8 +60,11 @@ class FileIndexMonitor;
class ftController;
class ftExtraList;
class ftFileSearch;
class ftServer: public RsFiles
class ftDataMultiplex;
class ftServer: public RsFiles, public ftDataSend
{
public:
@ -68,58 +73,75 @@ class ftServer: public RsFiles
/******************** Setup ************************************/
/***************************************************************/
ftServer(CacheStrapper *cStrapper) { return; }
ftServer(CacheStrapper *cStrapper, p3ConnectMgr *connMgr);
/* Assign important variables */
void setConfigDirectory(std::string path);
void setPQInterface(PQInterface *pqi);
/* Final Setup (once everything is assigned) */
void SetupFtServer();
/* add Config Items (Extra, Controller) */
void addConfigComponents(p3ConfigMgr *mgr);
CacheStrapper *getCacheStrapper();
CacheTransfer *getCacheTransfer();
/* Final Setup (once everything is assigned) */
void SetupFtServer();
void StartupThreads();
/***************************************************************/
/*************** Control Interface *****************************/
/************** (Implements RsFiles) ***************************/
/***************************************************************/
/***
* Control of Downloads
***/
virtual bool FileRequest(std::string fname, std::string hash,
uint32_t size, std::string dest, uint32_t flags);
virtual bool FileCancel(std::string hash);
virtual bool FileControl(std::string hash, uint32_t flags);
virtual bool FileClearCompleted();
/* get Details of File Transfers */
/***
* Download/Upload Details
***/
virtual bool FileDownloads(std::list<std::string> &hashs);
virtual bool FileUploads(std::list<std::string> &hashs);
virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info);
/* Access ftExtraList - Details */
/***
* Extra List Access
***/
virtual bool ExtraFileAdd(std::string fname, std::string hash, uint32_t size,
uint32_t period, uint32_t flags);
virtual bool ExtraFileRemove(std::string hash, uint32_t flags);
virtual bool ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags);
virtual bool ExtraFileHash(std::string localpath,
uint32_t period, uint32_t flags);
virtual bool ExtraFileStatus(std::string localpath, FileInfo &info);
/* Directory Listing */
/***
* Directory Listing / Search Interface
***/
virtual int RequestDirDetails(std::string uid, std::string path, DirDetails &details);
virtual int RequestDirDetails(void *ref, DirDetails &details, uint32_t flags);
/* Search Interface */
virtual int SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results);
virtual int SearchBoolExp(Expression * exp, std::list<FileDetail> &results);
/* Utility Functions */
/***
* Utility Functions
***/
virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath);
virtual void ForceDirectoryCheck();
virtual bool InDirectoryCheck();
/* Directory Handling */
/***
* Directory Handling
***/
virtual void setDownloadDirectory(std::string path);
virtual void setPartialsDirectory(std::string path);
virtual std::string getDownloadDirectory();
@ -129,8 +151,6 @@ virtual bool getSharedDirectories(std::list<std::string> &dirs);
virtual bool addSharedDirectory(std::string dir);
virtual bool removeSharedDirectory(std::string dir);
virtual int reScanDirs();
virtual int check_dBUpdate();
/***************************************************************/
/*************** Control Interface *****************************/
@ -139,9 +159,20 @@ virtual int check_dBUpdate();
/***************************************************************/
/*************** Data Transfer Interface ***********************/
/***************************************************************/
public:
virtual bool sendData(std::string peerId, std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize, void *data);
virtual bool sendDataRequest(std::string peerId,
std::string hash, uint64_t size,
uint64_t offset, uint32_t chunksize);
bool requestData(std::string peerId, std::string hash,
uint64_t offset, uint32_t size);
/*************** Internal Transfer Fns *************************/
virtual int tick();
private:
bool handleInputQueues();
bool handleCacheData();
bool handleFileData();
/******************* p3 Config Overload ************************/
protected:
@ -155,18 +186,34 @@ bool loadConfigMap(std::map<std::string, std::string> &configMap);
/******************* p3 Config Overload ************************/
/*************************** p3 Config Overload ********************/
private:
/**** INTERNAL FUNCTIONS ***/
virtual int reScanDirs();
virtual int check_dBUpdate();
private:
/* no need for Mutex protection -
* as each component is protected independently.
*/
P3Interface *mP3iface; /* XXX THIS NEEDS PROTECTION */
p3AuthMgr *mAuthMgr;
p3ConnectMgr *mConnMgr;
CacheStrapper *mCacheStrapper;
ftController *mFtController;
ftExtraList *mFtExtra;
FileIndexStore *fiStore;
FileIndexMonitor *fimon;
ftDataMultiplex *mFtDataplex;
FileIndexStore *mFiStore;
FileIndexMonitor *mFiMon;
ftFileSearch *mFtSearch;
RsMutex srvMutex;
std::string mConfigPath;

64
libretroshare/src/ft/fttransfermodule.cc
View File

@ -32,9 +32,17 @@ const uint32_t PQIPEER_DOWNLOAD_TOO_FAST = 8; /* 8 secs */
const uint32_t PQIPEER_DOWNLOAD_TOO_SLOW = 12; /* 12 secs */
const uint32_t PQIPEER_DOWNLOAD_MIN_DELTA = 5; /* 5 secs */
ftTransferModule::ftTransferModule(ftFileCreator *fc,ftClientModule *cm)
:mFileCreator(fc),mClientModule(cm),mFlag(0)
{}
ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm)
:mFileCreator(fc), mMultiplexor(dm), mFlag(0)
{
mHash = mFileCreator->getHash();
mSize = mFileCreator->getFileSize();
// Dummy for Testing (should be handled independantly for
// each peer.
mChunkSize = 10000;
return;
}
ftTransferModule::~ftTransferModule()
{}
@ -50,12 +58,12 @@ bool ftTransferModule::setFileSources(std::list<std::string> peerIds)
return 1;
}
bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t maxRate); //state = ONLINE/OFFLINE
bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t maxRate) //state = ONLINE/OFFLINE
{
bool found = false;
std::list<std::string>::iterator it;
it = peerIds.begin();
while (( it != peerIds.end())&&(!found))
it = mFileSources.begin();
while (( it != mFileSources.end())&&(!found))
{
if ((*it) == peerId)
found = true;
@ -66,12 +74,12 @@ bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t m
std::map<std::string,peerInfo>::iterator mit;
mit = mOnlinePeers.find(peerId);
if (mit == map::end)
if (mit == mOnlinePeers.end())
{
peerInfo pInfo;
pInfo.state = state;
pInfo.desiredRate = maxRate;
mOnlinePeers.insert(pair<std::string,peerInfo>(peerId,pInfo));
mOnlinePeers[peerId] = pInfo;
}
else
{
@ -86,15 +94,31 @@ uint32_t ftTransferModule::getDataRate(std::string peerId)
{
std::map<std::string,peerInfo>::iterator mit;
mit = mOnlinePeers.find(peerId);
if (mit == map::end)
if (mit == mOnlinePeers.end())
return 0;
else
return (mit->second).actualRate;
}
void ftTransferModule::requestData(uint64_t offset, uint32_t chunk_size)
//interface to client module
bool ftTransferModule::recvFileData(std::string peerId, uint64_t offset,
uint32_t chunk_size, void *data)
{
mClientModule->requestData(mHash,offset,chunk_size);
#ifdef FT_DEBUG
std::cerr << "ftTransferModule::recvFileData()";
std::cerr << " peerId: " << peerId;
std::cerr << " offset: " << offset;
std::cerr << " chunksize: " << chunk_size;
std::cerr << std::endl;
#endif
return storeData(offset, chunk_size, data);
}
void ftTransferModule::requestData(std::string peerId, uint64_t offset, uint32_t chunk_size)
{
mMultiplexor->sendDataRequest(peerId, mHash, mSize, offset,chunk_size);
}
bool ftTransferModule::getChunk(uint64_t &offset, uint32_t &chunk_size)
@ -104,7 +128,7 @@ bool ftTransferModule::getChunk(uint64_t &offset, uint32_t &chunk_size)
bool ftTransferModule::storeData(uint64_t offset, uint32_t chunk_size,void *data)
{
mFileCreator->storeData(offset, chunk_size, data);
mFileCreator -> addFileData(offset, chunk_size, data);
}
void ftTransferModule::queryInactive()
@ -133,7 +157,7 @@ void ftTransferModule::queryInactive()
(mit->second).req_size = mChunkSize;
(mit->second).lastTS = ts;
(mit->second).state = PQIPEER_DOWNLOADING;
requestData(mOffset,mChunkSize);
requestData(mit->first, mOffset,mChunkSize);
}
else mFlag = 1;
}
@ -142,7 +166,7 @@ void ftTransferModule::queryInactive()
//file request has been sent to peer side, but no response received yet
case PQIPEER_DOWNLOADING:
if (ts - ((mit->second).lastTS) > PQIPEER_DOWNLOAD_CHECK)
requestData((mit->second).req_loc,(mit->second).req_size); //give a push
requestData(mit->first, (mit->second).req_loc,(mit->second).req_size); //give a push
break;
//file response has been received or peer side is just ready for download
@ -154,7 +178,7 @@ void ftTransferModule::queryInactive()
(mit->second).req_size = mChunkSize;
(mit->second).lastTS = ts;
(mit->second).state = PQIPEER_DOWNLOADING;
requestData(mOffset,mChunkSize);
requestData(mit->first, mOffset,mChunkSize);
}
else mFlag = 1;
break;
@ -192,7 +216,7 @@ bool ftTransferModule::resumeTransfer()
return 1;
}
void ftTransferModule::completeFileTransfer()
bool ftTransferModule::completeFileTransfer()
{
}
@ -205,3 +229,11 @@ int ftTransferModule::tick()
return 0;
}
void ftTransferModule::adjustSpeed()
{
return;
}

40
libretroshare/src/ft/fttransfermodule.h
View File

@ -37,7 +37,12 @@
#include <map>
#include <list>
#include <ftFileCreator.h>
#include <string>
#include "ft/ftfilecreator.h"
#include "ft/ftdatamultiplex.h"
#include "util/rsthreads.h"
const int PQIPEER_INIT = 0x0000;
const int PQIPEER_NOT_ONLINE = 0x0001;
@ -47,13 +52,15 @@ const int PQIPEER_SUSPEND = 0x0010;
class Request
{
public:
uint64_t offset;
uint32_t chunkSize;
};
class peerInfo
{
std:string peerId;
public:
std::string peerId;
uint32_t state;
uint32_t desiredRate;
Request lastRequest;
@ -69,7 +76,7 @@ class peerInfo
class ftTransferModule
{
public:
ftTransferModule(ftFileCreator *fc,ftClientModule *cm);
ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm);
~ftTransferModule();
//interface to download controller
@ -80,30 +87,35 @@ public:
bool resumeTransfer();
bool completeFileTransfer();
//interface to client module
bool recvFileData(uint64_t offset, uint32_t chunk_size, void *data); //called by ftClientModule when data arrives
void requestData(uint64_t offset, uint32_t chunk_size);
//interface to multiplex module
bool recvFileData(std::string peerId, uint64_t offset,
uint32_t chunk_size, void *data);
void requestData(std::string peerId, uint64_t offset, uint32_t chunk_size);
//interface to file creator , just wrapper functions that call the ftFileCreator
//interface to file creator
bool getChunk(uint64_t &offset, uint32_t &chunk_size);
bool storeData(uint64_t offset, uint32_t chunk_size, void *data);
//internal used functions
void queryInactive();
int tick();
/* add by DrBob for interfaces */
std::string hash() { return mHash; }
uint64_t size() { return mSize; }
public:
ftFileCreator *mFileCreator;
ftClientModule *mClientModule;
//internal used functions
void queryInactive();
void adjustSpeed();
private:
/* These have independent Mutexes / are const locally (no Mutex protection)*/
ftFileCreator *mFileCreator;
ftDataMultiplex *mMultiplexor;
std::string mHash;
uint64_t mSize;
RsMutex tfMtx; /* below is mutex protected */
std::list<std::string> mFileSources;
std::map<std::string,peerInfo> mOnlinePeers;

77
libretroshare/src/rsiface/rsfiles.h
View File

@ -51,19 +51,22 @@ const uint32_t RS_FILE_CTRL_STREAM_VIDEO = 0x0006;
/************************************
* Used To indicate where to search.
*
* The Order of these is very important,
* it specifies the search order too.
*
*/
const uint32_t RS_FILE_HINTS_MASK = 0x00ff;
const uint32_t RS_FILE_HINTS_MASK = 0x00ffffff;
const uint32_t RS_FILE_HINTS_CACHE = 0x0001;
const uint32_t RS_FILE_HINTS_EXTRA = 0x0002;
const uint32_t RS_FILE_HINTS_LOCAL = 0x0004;
const uint32_t RS_FILE_HINTS_REMOTE = 0x0008;
const uint32_t RS_FILE_HINTS_DOWNLOAD = 0x0010;
const uint32_t RS_FILE_HINTS_UPLOAD = 0x0020;
const uint32_t RS_FILE_HINTS_SPEC_ONLY = 0x1000;
const uint32_t RS_FILE_HINTS_CACHE = 0x00000001;
const uint32_t RS_FILE_HINTS_EXTRA = 0x00000002;
const uint32_t RS_FILE_HINTS_LOCAL = 0x00000004;
const uint32_t RS_FILE_HINTS_REMOTE = 0x00000008;
const uint32_t RS_FILE_HINTS_DOWNLOAD = 0x00000010;
const uint32_t RS_FILE_HINTS_UPLOAD = 0x00000020;
const uint32_t RS_FILE_HINTS_SPEC_ONLY = 0x01000000;
const uint32_t RS_FILE_EXTRA_DELETE = 0x0010;
@ -79,43 +82,27 @@ virtual ~RsFiles() { return; }
/****************************************/
/* download */
/* Required Interfaces ......
*
* 1) Access to downloading / uploading files.
*/
/* get Details of File Transfers */
virtual bool FileDownloads(std::list<std::string> &hashs) = 0;
virtual bool FileUploads(std::list<std::string> &hashs) = 0;
virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0;
/*
* 2) Control of Downloads.
*
*/
/***
* Control of Downloads.
***/
virtual bool FileRequest(std::string fname, std::string hash,
uint32_t size, std::string dest, uint32_t flags) = 0;
virtual bool FileCancel(std::string hash) = 0;
virtual bool FileControl(std::string hash, uint32_t flags) = 0;
virtual bool FileClearCompleted() = 0;
/***
* Download / Upload Details.
***/
virtual bool FileDownloads(std::list<std::string> &hashs) = 0;
virtual bool FileUploads(std::list<std::string> &hashs) = 0;
virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0;
/*
* 3) Addition of Extra Files... From File System
* These are Hashed and stored in the 'Hidden Files' section
* which can only be accessed if you know the hash.
*
* FileHash is called to start the hashing process,
* and add the file to the HiddenStore.
*
* FileHashStatus is called to lookup files
* and see if the hashing is completed.
*/
/* Access ftExtraList - Details */
/***
* Extra List Access
***/
virtual bool ExtraFileAdd(std::string fname, std::string hash, uint32_t size,
uint32_t period, uint32_t flags) = 0;
virtual bool ExtraFileRemove(std::string hash, uint32_t flags) = 0;
@ -124,25 +111,25 @@ virtual bool ExtraFileHash(std::string localpath,
virtual bool ExtraFileStatus(std::string localpath, FileInfo &info) = 0;
/*
* 4) Search and Listing Interface
/***
* Directory Listing / Search Interface
*/
/* Directory Listing / Search Interface */
virtual int RequestDirDetails(std::string uid, std::string path, DirDetails &details) = 0;
virtual int RequestDirDetails(void *ref, DirDetails &details, uint32_t flags) = 0;
virtual int SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results) = 0;
virtual int SearchBoolExp(Expression * exp, std::list<FileDetail> &results) = 0;
/*
* 5) Directory Control / Shared Files Utility Functions.
*/
/***
* Utility Functions.
***/
virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath) = 0;
virtual void ForceDirectoryCheck() = 0;
virtual bool InDirectoryCheck() = 0;
/***
* Directory Control
***/
virtual void setDownloadDirectory(std::string path) = 0;
virtual void setPartialsDirectory(std::string path) = 0;
virtual std::string getDownloadDirectory() = 0;

9
libretroshare/src/serialiser/rstlvtypes.cc
View File

@ -33,6 +33,10 @@
#include <iomanip>
#include <iostream>
void RsTlvItem::TlvShallowClear()
{
TlvClear(); /* unless overloaded! */
}
std::ostream &RsTlvItem::printBase(std::ostream &out, std::string clsName, uint16_t indent)
{
@ -102,6 +106,11 @@ void RsTlvBinaryData::TlvClear()
{
free(bin_data);
}
TlvShallowClear();
}
void RsTlvBinaryData::TlvShallowClear()
{
bin_data = NULL;
bin_len = 0;
}

3
libretroshare/src/serialiser/rstlvtypes.h
View File

@ -47,6 +47,7 @@ class RsTlvItem
virtual ~RsTlvItem() { return; }
virtual uint16_t TlvSize() = 0;
virtual void TlvClear() = 0;
virtual void TlvShallowClear(); /*! Don't delete allocated data */
virtual bool SetTlv(void *data, uint32_t size, uint32_t *offset) = 0; /* serialise */
virtual bool GetTlv(void *data, uint32_t size, uint32_t *offset) = 0; /* deserialise */
virtual std::ostream &print(std::ostream &out, uint16_t indent) = 0;
@ -66,6 +67,8 @@ class RsTlvBinaryData: public RsTlvItem
virtual ~RsTlvBinaryData();
virtual uint16_t TlvSize();
virtual void TlvClear(); /*! Initialize fields to empty legal values ( "0", "", etc) */
virtual void TlvShallowClear(); /*! Don't delete the binary data */
/// Serialise.
/*! Serialise Tlv to buffer(*data) of 'size' bytes starting at *offset */
virtual bool SetTlv(void *data, uint32_t size, uint32_t *offset);

6
libretroshare/src/util/rsthreads.h
View File

@ -76,13 +76,17 @@ virtual void run() = 0; /* called once the thread is started */
};
class RsQueueThread
class RsQueueThread: public RsThread
{
public:
RsQueueThread(uint32_t min, uint32_t max, double relaxFactor );
virtual ~RsQueueThread() { return; }
virtual void run();
protected:
virtual bool workQueued() = 0;
virtual bool doWork() = 0;