From 5c6e558942fa41ab436201f757fab790b64b3828 Mon Sep 17 00:00:00 2001 From: drbob Date: Sun, 3 Aug 2008 12:45:53 +0000 Subject: [PATCH] More Improvements to FileTransfer: * Added TlvShallowClear() to serialisers * Added RsQueueThread for periodic queue processes. * Completed ftDataMultiplex which replaces ftServer/ClientModules. * Added Server Queue / Thread to ftDataMultiplex. * Added ftdataplextest to exercise ftDataMultiplex. * Generalised ftFileSearch to handle an array of ftSearch classes. * Tweaked rsfiles.h #defines to match new ftFileSearch scheme. * Added Generic ftData Interfaces for Testing. * added ftDataSend/Recv Interfaces to ftServer + ftDataMultiplex respectively. * Completed much of ftServer (External Interface), but not yet done. * Extra debugging and small changes to ftExtraList * Integrated new ftTransferModule with minor interface changes. git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@660 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/ft/Makefile | 12 +- libretroshare/src/ft/ftcontroller.h | 8 +- libretroshare/src/ft/ftdata.cc | 82 ++++ libretroshare/src/ft/ftdata.h | 68 +++- libretroshare/src/ft/ftdatamultiplex.cc | 140 ++++++- libretroshare/src/ft/ftdatamultiplex.h | 36 +- libretroshare/src/ft/ftdataplextest.cc | 174 ++++++++ libretroshare/src/ft/ftextralist.cc | 88 +++- libretroshare/src/ft/ftextralist.h | 10 +- libretroshare/src/ft/ftextralisttest.cc | 2 +- libretroshare/src/ft/ftfileprovider.cc | 10 + libretroshare/src/ft/ftfilesearch.cc | 165 ++++++-- libretroshare/src/ft/ftfilesearch.h | 25 +- libretroshare/src/ft/ftsearch.h | 12 +- libretroshare/src/ft/ftserver.cc | 453 ++++++++++++++++++--- libretroshare/src/ft/ftserver.h | 85 +++- libretroshare/src/ft/fttransfermodule.cc | 98 +++-- libretroshare/src/ft/fttransfermodule.h | 118 +++--- libretroshare/src/rsiface/rsfiles.h | 77 ++-- libretroshare/src/serialiser/rstlvtypes.cc | 9 + libretroshare/src/serialiser/rstlvtypes.h | 3 + libretroshare/src/util/rsthreads.h | 6 +- 22 files changed, 1355 insertions(+), 326 deletions(-) create mode 100644 libretroshare/src/ft/ftdata.cc create mode 100644 libretroshare/src/ft/ftdataplextest.cc diff --git a/libretroshare/src/ft/Makefile b/libretroshare/src/ft/Makefile index a9947051b..41810f7ed 100644 --- a/libretroshare/src/ft/Makefile +++ b/libretroshare/src/ft/Makefile @@ -7,12 +7,13 @@ RS_TOP_DIR = .. include $(RS_TOP_DIR)/scripts/config.mk ############################################################### -RSOBJ = ftfileprovider.o ftfilecreator.o ftextralist.o ftdatamultiplex.o -#ftservermodule.o +RSOBJ = ftdata.o ftfileprovider.o ftfilecreator.o ftextralist.o ftdatamultiplex.o ftfilesearch.o fttransfermodule.o -TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o +#ftserver.o -TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest +TESTOBJ = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o + +TESTS = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest all: librs tests @@ -25,6 +26,9 @@ ftfileprovidertest : ftfileprovidertest.o ftextralisttest : ftextralisttest.o $(CC) $(CFLAGS) -o ftextralisttest ftextralisttest.o $(LIBS) +ftdataplextest : ftdataplextest.o + $(CC) $(CFLAGS) -o ftdataplextest ftdataplextest.o $(LIBS) + ############################################################### include $(RS_TOP_DIR)/scripts/rules.mk ############################################################### diff --git a/libretroshare/src/ft/ftcontroller.h b/libretroshare/src/ft/ftcontroller.h index af37ffb28..3352df723 100644 --- a/libretroshare/src/ft/ftcontroller.h +++ b/libretroshare/src/ft/ftcontroller.h @@ -38,6 +38,10 @@ * */ +class ftFileCreator; +class ftTransferModule; +class ftFileProvider; + class ftController: public CacheTransfer, public RsThread, public pqiMonitor, public p3Config { public: @@ -96,8 +100,8 @@ virtual bool loadList(std::list load); RsMutex ctrlMutex; - std::list incomingQueue; - std::map mCompleted; + std::list incomingQueue; + std::map mCompleted; std::map mTransfers; std::map mFileCreators; diff --git a/libretroshare/src/ft/ftdata.cc b/libretroshare/src/ft/ftdata.cc new file mode 100644 index 000000000..3094e2f84 --- /dev/null +++ b/libretroshare/src/ft/ftdata.cc @@ -0,0 +1,82 @@ +/* + * libretroshare/src/ft: ftdata.cc + * + * File Transfer for RetroShare. + * + * Copyright 2008 by Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include "ft/ftdata.h" + +/******* Pair of Send/Recv (Only need to handle Send side) ******/ +ftDataSendPair::ftDataSendPair(ftDataRecv *recv) + :mDataRecv(recv) +{ + return; +} + + /* Client Send */ +bool ftDataSendPair::sendDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize) +{ + return mDataRecv->recvDataRequest(peerId,hash,size,offset,chunksize); +} + + /* Server Send */ +bool ftDataSendPair::sendData(std::string peerId, + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) +{ + return mDataRecv->recvData(peerId, hash,size,offset,chunksize,data); +} + + + /* Client Send */ +bool ftDataSendDummy::sendDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize) +{ + return true; +} + + /* Server Send */ +bool ftDataSendDummy::sendData(std::string peerId, + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) +{ + return true; +} + + + /* Client Recv */ +bool ftDataRecvDummy::recvData(std::string peerId, + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) +{ + return true; +} + + + /* Server Recv */ +bool ftDataRecvDummy::recvDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize) +{ + return true; +} + diff --git a/libretroshare/src/ft/ftdata.h b/libretroshare/src/ft/ftdata.h index 6792745be..52c4581a2 100644 --- a/libretroshare/src/ft/ftdata.h +++ b/libretroshare/src/ft/ftdata.h @@ -46,12 +46,12 @@ class ftDataSend virtual ~ftDataSend() { return; } /* Client Send */ -virtual bool sendDataRequest(std::string peerId, std::string hash, - uint64_t offset, uint32_t size) = 0; +virtual bool sendDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize) = 0; /* Server Send */ -virtual bool sendData(std::string peerId, std::string hash, - uint64_t offset, uint32_t size, void *data) = 0; +virtual bool sendData(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) = 0; }; @@ -66,12 +66,64 @@ class ftDataRecv virtual ~ftDataRecv() { return; } /* Client Recv */ -virtual bool recvData(std::string peerId, std::string hash, - uint64_t offset, uint32_t size, void *data) = 0; +virtual bool recvData(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) = 0; /* Server Recv */ -virtual bool recvDataRequest(std::string peerId, std::string hash, - uint64_t offset, uint32_t size) = 0; +virtual bool recvDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize) = 0; + + +}; + +/******* Pair of Send/Recv (Only need to handle Send side) ******/ +class ftDataSendPair: public ftDataSend +{ + public: + + ftDataSendPair(ftDataRecv *recv); +virtual ~ftDataSendPair() { return; } + + /* Client Send */ +virtual bool sendDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize); + + /* Server Send */ +virtual bool sendData(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data); + + ftDataRecv *mDataRecv; +}; + + +class ftDataSendDummy: public ftDataSend +{ + public: +virtual ~ftDataSendDummy() { return; } + + /* Client Send */ +virtual bool sendDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize); + + /* Server Send */ +virtual bool sendData(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data); + +}; + +class ftDataRecvDummy: public ftDataRecv +{ + public: + +virtual ~ftDataRecvDummy() { return; } + + /* Client Recv */ +virtual bool recvData(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data); + + /* Server Recv */ +virtual bool recvDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize); }; diff --git a/libretroshare/src/ft/ftdatamultiplex.cc b/libretroshare/src/ft/ftdatamultiplex.cc index 4f4c78310..f94503d28 100644 --- a/libretroshare/src/ft/ftdatamultiplex.cc +++ b/libretroshare/src/ft/ftdatamultiplex.cc @@ -35,6 +35,11 @@ #include "ft/ftfileprovider.h" #include "ft/ftsearch.h" +/* For Thread Behaviour */ +const uint32_t DMULTIPLEX_MIN = 10; /* 1ms sleep */ +const uint32_t DMULTIPLEX_MAX = 1000; /* 1 sec sleep */ +const double DMULTIPLEX_RELAX = 0.5; /* ??? */ + ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator) :mModule(module), mCreator(creator) { @@ -44,15 +49,16 @@ ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator) const uint32_t FT_DATA = 0x0001; const uint32_t FT_DATA_REQ = 0x0002; -ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t offset, uint32_t chunk, void *data) - :mType(type), mPeerId(peerId), mHash(hash), +ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data) + :mType(type), mPeerId(peerId), mHash(hash), mSize(size), mOffset(offset), mChunk(chunk), mData(data) { return; } ftDataMultiplex::ftDataMultiplex(ftDataSend *server, ftSearch *search) - :mDataSend(server), mSearch(search) + :RsQueueThread(DMULTIPLEX_MIN, DMULTIPLEX_MAX, DMULTIPLEX_RELAX), + mDataSend(server), mSearch(search) { return; } @@ -90,49 +96,135 @@ bool ftDataMultiplex::removeTransferModule(ftTransferModule *mod, ftFileCreator /*************** SEND INTERFACE (calls ftDataSend) *******************/ /* Client Send */ -bool ftDataMultiplex::sendDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size) +bool ftDataMultiplex::sendDataRequest(std::string peerId, + std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize) { - return mDataSend->sendDataRequest(peerId, hash, offset, size); + return mDataSend->sendDataRequest(peerId,hash,size,offset,chunksize); } /* Server Send */ -bool ftDataMultiplex::sendData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data) +bool ftDataMultiplex::sendData(std::string peerId, + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) { - return mDataSend->sendData(peerId, hash, offset, size, data); + return mDataSend->sendData(peerId,hash,size,offset,chunksize,data); } /*************** RECV INTERFACE (provides ftDataRecv) ****************/ /* Client Recv */ -bool ftDataMultiplex::recvData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data) +bool ftDataMultiplex::recvData(std::string peerId, + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) { /* Store in Queue */ RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ mRequestQueue.push_back( - ftRequest(FT_DATA, peerId, hash, offset, size, data)); + ftRequest(FT_DATA,peerId,hash,size,offset,chunksize,data)); return true; } /* Server Recv */ -bool ftDataMultiplex::recvDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size) +bool ftDataMultiplex::recvDataRequest(std::string peerId, + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize) { /* Store in Queue */ RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ mRequestQueue.push_back( - ftRequest(FT_DATA_REQ, peerId, hash, offset, size, NULL)); + ftRequest(FT_DATA_REQ,peerId,hash,size,offset,chunksize,NULL)); return true; } /*********** BACKGROUND THREAD OPERATIONS ***********/ +bool ftDataMultiplex::workQueued() +{ + RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ + if (mRequestQueue.size() > 0) + { + return true; + } + + if (mSearchQueue.size() > 0) + { + return true; + } + + return false; +} + +bool ftDataMultiplex::doWork() +{ + bool doRequests = true; + + /* Handle All the current Requests */ + while(doRequests) + { + ftRequest req; + + { + RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ + if (mRequestQueue.size() == 0) + { + doRequests = false; + continue; + } + + req = mRequestQueue.front(); + mRequestQueue.pop_front(); + } + + /* MUTEX FREE */ + + switch(req.mType) + { + case FT_DATA: + handleRecvData(req.mPeerId, req.mHash, req.mSize, + req.mOffset, req.mChunk, req.mData); + break; + + case FT_DATA_REQ: + handleRecvDataRequest(req.mPeerId, req.mHash, + req.mSize, req.mOffset, req.mChunk); + break; + + default: + break; + } + } + + /* Only Handle One Search Per Period.... + * Lower Priority + */ + ftRequest req; + + { + RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ + if (mSearchQueue.size() == 0) + { + /* Finished */ + return true; + } + + req = mSearchQueue.front(); + mSearchQueue.pop_front(); + } + + handleSearchRequest(req.mPeerId, req.mHash, req.mSize, + req.mOffset, req.mChunk); + + return true; +} bool ftDataMultiplex::handleRecvData(std::string peerId, - std::string hash, uint64_t offset, uint32_t size, void *data) + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data) { RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/ std::map::iterator it; @@ -142,7 +234,7 @@ bool ftDataMultiplex::handleRecvData(std::string peerId, return false; } - (it->second).mModule->recvFileData(peerId, offset, size, data); + (it->second).mModule->recvFileData(peerId, offset, chunksize, data); return true; } @@ -150,7 +242,8 @@ bool ftDataMultiplex::handleRecvData(std::string peerId, /* called by ftTransferModule */ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId, - std::string hash, uint64_t offset, uint32_t size) + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize) { /**** Find Files *****/ @@ -159,7 +252,7 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId, if (mClients.end() != (cit = mClients.find(hash))) { locked_handleServerRequest((cit->second).mCreator, - peerId, hash, offset, size); + peerId, hash, size, offset, chunksize); return true; } @@ -167,26 +260,28 @@ bool ftDataMultiplex::handleRecvDataRequest(std::string peerId, if (mServers.end() != (sit = mServers.find(hash))) { locked_handleServerRequest(sit->second, - peerId, hash, offset, size); + peerId, hash, size, offset, chunksize); return true; } /* Add to Search Queue */ mSearchQueue.push_back( - ftRequest(FT_DATA_REQ, peerId, hash, offset, size, NULL)); + ftRequest(FT_DATA_REQ, peerId, hash, + size, offset, chunksize, NULL)); return true; } bool ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider, - std::string peerId, std::string hash, uint64_t offset, uint32_t size) + std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize) { void *data = malloc(size); - if (provider->getFileData(offset, size, data)) + if (provider->getFileData(offset, chunksize, data)) { /* send data out */ - sendData(peerId, hash, offset, size, data); + sendData(peerId, hash, size, offset, chunksize, data); return true; } return false; @@ -194,7 +289,8 @@ bool ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider, bool ftDataMultiplex::handleSearchRequest(std::string peerId, - std::string hash, uint64_t offset, uint32_t size) + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize) { { @@ -235,7 +331,7 @@ bool ftDataMultiplex::handleSearchRequest(std::string peerId, /* handle request finally */ locked_handleServerRequest(provider, - peerId, hash, offset, size); + peerId, hash, size, offset, chunksize); return true; } return false; diff --git a/libretroshare/src/ft/ftdatamultiplex.h b/libretroshare/src/ft/ftdatamultiplex.h index fd1ca4723..6c30a5ea2 100644 --- a/libretroshare/src/ft/ftdatamultiplex.h +++ b/libretroshare/src/ft/ftdatamultiplex.h @@ -61,11 +61,15 @@ class ftRequest { public: - ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t offset, uint32_t chunk, void *data); + ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data); + + ftRequest() + :mType(0), mSize(0), mOffset(0), mChunk(0), mData(NULL) { return; } uint32_t mType; std::string mPeerId; std::string mHash; + uint64_t mSize; uint64_t mOffset; uint32_t mChunk; void *mData; @@ -73,7 +77,7 @@ class ftRequest -class ftDataMultiplex: public ftDataRecv +class ftDataMultiplex: public ftDataRecv, public RsQueueThread { public: @@ -89,36 +93,48 @@ bool removeTransferModule(ftTransferModule *mod, ftFileCreator *f); /*************** SEND INTERFACE (calls ftDataSend) *******************/ /* Client Send */ -bool sendDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size); +bool sendDataRequest(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize); /* Server Send */ -bool sendData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data); +bool sendData(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data); /*************** RECV INTERFACE (provides ftDataRecv) ****************/ /* Client Recv */ -virtual bool recvData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data); +virtual bool recvData(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data); /* Server Recv */ -virtual bool recvDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size); +virtual bool recvDataRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize); + protected: + + /* Overloaded from RsQueueThread */ +virtual bool workQueued(); +virtual bool doWork(); + private: /* Handling Job Queues */ bool handleRecvData(std::string peerId, - std::string hash, uint64_t offset, uint32_t size, void *data); + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data); bool handleRecvDataRequest(std::string peerId, - std::string hash, uint64_t offset, uint32_t size); + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize); bool handleSearchRequest(std::string peerId, - std::string hash, uint64_t offset, uint32_t size); + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize); /* We end up doing the actual server job here */ bool locked_handleServerRequest(ftFileProvider *provider, - std::string peerId, std::string hash, uint64_t offset, uint32_t size); + std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize); RsMutex dataMtx; diff --git a/libretroshare/src/ft/ftdataplextest.cc b/libretroshare/src/ft/ftdataplextest.cc new file mode 100644 index 000000000..2b863529a --- /dev/null +++ b/libretroshare/src/ft/ftdataplextest.cc @@ -0,0 +1,174 @@ +/* + * libretroshare/src/ft: ftextralisttest.cc + * + * File Transfer for RetroShare. + * + * Copyright 2008 by Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +/* + * Test for Multiplexor..... + * As this is a key middle component, it is hard to test without other bits. + * It relies on ftFileProvider/ftFileCreator/ftTransferModule... + * + * And has dummy ftDataSend and ftSearch. + * + */ + +#include "ft/ftextralist.h" +#include "ft/ftdatamultiplex.h" +#include "ft/ftfilesearch.h" + +void do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list &files); + + +void usage(char *name) +{ + std::cerr << "Usage: " << name << " [-p ] [-d ] [ ... ] "; + std::cerr << std::endl; +} + +int main(int argc, char **argv) +{ + int c; + uint32_t period = 1; + uint32_t dPeriod = 600; /* default 10 minutes */ + + std::list fileList; + + while(-1 != (c = getopt(argc, argv, "d:p:"))) + { + switch (c) + { + case 'p': + period = atoi(optarg); + break; + case 'd': + dPeriod = atoi(optarg); + break; + default: + usage(argv[0]); + break; + } + } + + if (optind >= argc) + { + std::cerr << "Missing Files" << std::endl; + usage(argv[0]); + } + + for(; optind < argc; optind++) + { + std::cerr << "Adding: " << argv[optind] << std::endl; + fileList.push_back(std::string(argv[optind])); + } + + ftExtraList *eList = new ftExtraList(); + eList->start(); + + ftSearch *ftsd = new ftSearchDummy(); + ftFileSearch *ftfs = new ftFileSearch(); + + ftfs-> addSearchMode(ftsd, RS_FILE_HINTS_LOCAL); + ftfs-> addSearchMode(eList, RS_FILE_HINTS_EXTRA); + + ftDataSend *ftds = new ftDataSendDummy(); + + /* setup Actual Test bit */ + ftDataMultiplex *ftmplex = new ftDataMultiplex(ftds, ftfs); + ftmplex->start(); + + /* Setup Search with some valid results */ + + + /* Request Data */ + + /* now work the thread */ + std::list::iterator it; + uint32_t flags = 0; + for(it = fileList.begin(); it != fileList.end(); it++) + { + eList->hashExtraFile(*it, dPeriod, flags); + } + + + /* now request files from ftDataMultiplex */ + + /* just request random data packets first */ + do_random_server_test(ftmplex, eList, fileList); +} + + + +uint32_t do_random_server_iteration(ftDataMultiplex *mplex, ftExtraList *eList, std::list &files) +{ + std::cerr << "do_random_server_iteration()"; + std::cerr << std::endl; + + std::list::iterator it; + uint32_t i = 0; + for(it = files.begin(); it != files.end(); it++) + { + FileInfo info; + if (eList->hashExtraFileDone(*it, info)) + { + std::cerr << "Hash Done for: " << *it; + std::cerr << std::endl; + std::cerr << info << std::endl; + + std::cerr << "Requesting Data Packet"; + std::cerr << std::endl; + + /* Server Recv */ + uint64_t offset = 10000; + uint32_t chunk = 20000; + mplex->recvDataRequest("Peer", info.hash, info.size, offset, chunk); + + i++; + } + else + { + std::cerr << "do_random_server_iteration() Hash Not Done for: " << *it; + std::cerr << std::endl; + } + } + + return i; + +} + + +void do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list &files) +{ + std::cerr << "do_random_server_test()"; + std::cerr << std::endl; + + uint32_t size = files.size(); + while(size > do_random_server_iteration(mplex, eList, files)) + { + std::cerr << "do_random_server_test() sleep"; + std::cerr << std::endl; + + sleep(10); + } +} + + diff --git a/libretroshare/src/ft/ftextralist.cc b/libretroshare/src/ft/ftextralist.cc index bb8a79ee2..321dc333e 100644 --- a/libretroshare/src/ft/ftextralist.cc +++ b/libretroshare/src/ft/ftextralist.cc @@ -26,6 +26,8 @@ #include "ft/ftextralist.h" #include "util/rsdir.h" +#define DEBUG_ELIST 1 + ftExtraList::ftExtraList() :p3Config(CONFIG_FT_EXTRA_LIST) { @@ -41,6 +43,11 @@ void ftExtraList::run() while (1) { +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::run() Iteration"; + std::cerr << std::endl; +#endif + now = time(NULL); { @@ -76,15 +83,29 @@ void ftExtraList::run() void ftExtraList::hashAFile() { +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::hashAFile()"; + std::cerr << std::endl; +#endif + /* extract entry from the queue */ FileDetails details; { RsStackMutex stack(extMutex); + + if (mToHash.size() == 0) + return; + details = mToHash.front(); mToHash.pop_front(); } +#ifdef DEBUG_ELIST + std::cerr << "Hashing: " << details.info.path; + std::cerr << std::endl; +#endif + /* hash it! */ std::string name, hash; uint64_t size; @@ -110,6 +131,16 @@ void ftExtraList::hashAFile() bool ftExtraList::addExtraFile(std::string path, std::string hash, uint64_t size, uint32_t period, uint32_t flags) { +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::addExtraFile() path: " << path; + std::cerr << " hash: " << hash; + std::cerr << " size: " << size; + std::cerr << " period: " << period; + std::cerr << " flags: " << flags; + + std::cerr << std::endl; +#endif + RsStackMutex stack(extMutex); FileDetails details; @@ -126,11 +157,42 @@ bool ftExtraList::addExtraFile(std::string path, std::string hash, /* stick it in the available queue */ mFiles[details.info.hash] = details; + return true; } +bool ftExtraList::removeExtraFile(std::string hash, uint32_t flags) +{ +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::removeExtraFile()"; + std::cerr << " hash: " << hash; + std::cerr << " flags: " << flags; + + std::cerr << std::endl; +#endif + + RsStackMutex stack(extMutex); + + std::map::iterator it; + it = mFiles.find(hash); + if (it == mFiles.end()) + { + return false; + } + + mFiles.erase(it); + + return true; +} + + bool ftExtraList::cleanupOldFiles() { +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::cleanupOldFiles()"; + std::cerr << std::endl; +#endif + RsStackMutex stack(extMutex); time_t now = time(NULL); @@ -159,6 +221,7 @@ bool ftExtraList::cleanupOldFiles() } } } + return true; } @@ -169,6 +232,14 @@ bool ftExtraList::cleanupOldFiles() bool ftExtraList::hashExtraFile(std::string path, uint32_t period, uint32_t flags) { +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::hashExtraFile() path: " << path; + std::cerr << " period: " << period; + std::cerr << " flags: " << flags; + + std::cerr << std::endl; +#endif + /* add into queue */ RsStackMutex stack(extMutex); @@ -180,6 +251,11 @@ bool ftExtraList::hashExtraFile(std::string path, uint32_t period, uint32_t fla bool ftExtraList::hashExtraFileDone(std::string path, FileInfo &info) { +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::hashExtraFileDone()"; + std::cerr << std::endl; +#endif + std::string hash; { /* Find in the path->hash map */ @@ -192,19 +268,25 @@ bool ftExtraList::hashExtraFileDone(std::string path, FileInfo &info) } hash = it->second; } - return searchExtraFiles(hash, info); + return search(hash, 0, 0, info); } /*** * Search Function - used by File Transfer * **/ -bool ftExtraList::searchExtraFiles(std::string hash, FileInfo &info) +bool ftExtraList::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const { + +#ifdef DEBUG_ELIST + std::cerr << "ftExtraList::search()"; + std::cerr << std::endl; +#endif + RsStackMutex stack(extMutex); /* find hash */ - std::map::iterator fit; + std::map::const_iterator fit; if (mFiles.end() == (fit = mFiles.find(hash))) { return false; diff --git a/libretroshare/src/ft/ftextralist.h b/libretroshare/src/ft/ftextralist.h index 7c3ae9de1..3112f93ec 100644 --- a/libretroshare/src/ft/ftextralist.h +++ b/libretroshare/src/ft/ftextralist.h @@ -57,6 +57,7 @@ #include #include +#include "ft/ftsearch.h" #include "util/rsthreads.h" #include "rsiface/rsfiles.h" #include "pqi/p3cfgmgr.h" @@ -108,7 +109,7 @@ const uint32_t CONFIG_FT_EXTRA_LIST = 1; const uint32_t CLEANUP_PERIOD = 600; /* 10 minutes */ -class ftExtraList: public RsThread, public p3Config +class ftExtraList: public RsThread, public p3Config, public ftSearch { public: @@ -122,6 +123,8 @@ class ftExtraList: public RsThread, public p3Config bool addExtraFile(std::string path, std::string hash, uint64_t size, uint32_t period, uint32_t flags); +bool removeExtraFile(std::string hash, uint32_t flags); + /*** * Hash file, and add to the files, * file is removed after period. @@ -132,9 +135,10 @@ bool hashExtraFileDone(std::string path, FileInfo &info); /*** * Search Function - used by File Transfer + * implementation of ftSearch. * **/ -bool searchExtraFiles(std::string hash, FileInfo &info); +virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; /*** * Thread Main Loop @@ -156,7 +160,7 @@ virtual bool loadList(std::list load); void hashAFile(); bool cleanupOldFiles(); - RsMutex extMutex; + mutable RsMutex extMutex; std::list mToHash; diff --git a/libretroshare/src/ft/ftextralisttest.cc b/libretroshare/src/ft/ftextralisttest.cc index 88708a7cf..95fad037e 100644 --- a/libretroshare/src/ft/ftextralisttest.cc +++ b/libretroshare/src/ft/ftextralisttest.cc @@ -142,7 +142,7 @@ void displayExtraListDetails(ftExtraList *eList, std::list toHash, for(it = hashed.begin(); it != hashed.end(); it++) { FileInfo info; - if (eList->searchExtraFiles(*it, info)) + if (eList->search(*it, 0, 0, info)) { std::cerr << "displayExtraListDetails() Found Hash: " << *it; std::cerr << std::endl; diff --git a/libretroshare/src/ft/ftfileprovider.cc b/libretroshare/src/ft/ftfileprovider.cc index 387d7d992..dc9fd5f43 100644 --- a/libretroshare/src/ft/ftfileprovider.cc +++ b/libretroshare/src/ft/ftfileprovider.cc @@ -11,6 +11,16 @@ ftFileProvider::~ftFileProvider(){ } } +std::string ftFileProvider::getHash() +{ + return hash; +} + +uint64_t ftFileProvider::getFileSize() +{ + return total_size; +} + bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *data) { RsStackMutex stack(ftPMutex); /********** STACK LOCKED MTX ******/ diff --git a/libretroshare/src/ft/ftfilesearch.cc b/libretroshare/src/ft/ftfilesearch.cc index a20517af1..b52fdbc28 100644 --- a/libretroshare/src/ft/ftfilesearch.cc +++ b/libretroshare/src/ft/ftfilesearch.cc @@ -23,57 +23,142 @@ * */ -#ifndef FT_FILE_SEARCH_HEADERd -#define FT_FILE_SEARCH_HEADER - -/* - * ftFileSearch - * - * This is actually implements the ftSearch Interface. - * - */ - #include "ft/ftfilesearch.h" -#include "dbase/cachestrapper.h" -#include "dbase/fimonitor.h" -#include "dbase/fistore.h" -bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) +const uint32_t MAX_SEARCHS = 24; /* lower 24 bits of hint */ + +ftFileSearch::ftFileSearch() + :mSearchs(MAX_SEARCHS) { - -#if 0 - /* actual search depends on the hints */ - if (hintflags | CACHE) + uint32_t i; + for(i = 0; i < MAX_SEARCHS; i++) { - mCacheStrapper->.. + mSearchs[i] = NULL; } +} - if (hintflags | LOCAL) - { +bool ftFileSearch::addSearchMode(ftSearch *search, uint32_t hintflags) +{ + hintflags &= 0x00ffffff; - } - - if (hintflags | EXTRA) - { - - - } - - if (hintflags | REMOTE) - { - - } - - private: - - CacheStrapper *mCacheStrapper; - ftExtraList *mExtraList; - FileIndexMonitor *mFileMonitor; - FileIndexStore *mFileStore; +#ifndef DEBUG_SEARCH + std::cerr << "ftSearchDummy::addSearchMode() : " << hintflags; + std::cerr << std::endl; #endif + uint32_t i; + for (i = 0; i < MAX_SEARCHS; i++) + { + uint32_t hints = hintflags >> i; + if (hints & 0x0001) + { + /* has the flag */ + mSearchs[i] = search; + +#ifndef DEBUG_SEARCH + std::cerr << "ftSearchDummy::addSearchMode() to slot "; + std::cerr << i; + std::cerr << std::endl; +#endif + + return true; + } + } + +#ifndef DEBUG_SEARCH + std::cerr << "ftSearchDummy::addSearchMode() Failed"; + std::cerr << std::endl; +#endif + + return false; +} + +bool ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const +{ + uint32_t hints, i; + +#ifndef DEBUG_SEARCH + std::cerr << "ftFileSearch::search(" << hash << ", " << size; + std::cerr << ", " << hintflags << ");"; + std::cerr << std::endl; +#endif + + for (i = 0; i < MAX_SEARCHS; i++) + { + hints = hintflags >> i; + if (hints & 0x0001) + { + /* has the flag */ + ftSearch *search = mSearchs[i]; + if (search) + { +#ifndef DEBUG_SEARCH + std::cerr << "ftFileSearch::search() SLOT: "; + std::cerr << i; + std::cerr << std::endl; +#endif + if (search->search(hash, size, hintflags, info)) + { + return true; + } + } + } + } + + /* if we haven't found it by now! - check if SPEC_ONLY flag is set */ + if (hintflags & RS_FILE_HINTS_SPEC_ONLY) + { +#ifndef DEBUG_SEARCH + std::cerr << "ftFileSearch::search() SPEC_ONLY: Failed"; + std::cerr << std::endl; +#endif + return false; + } + +#ifndef DEBUG_SEARCH + std::cerr << "ftSearchDummy::search() Searching Others:"; + std::cerr << std::endl; +#endif + + /* if we don't have the SPEC_ONLY flag, + * we check through all the others + */ + for (i = 0; i < MAX_SEARCHS; i++) + { + hints = hintflags >> i; + if (hints & 0x0001) + { + continue; + } + + /* has the flag */ + ftSearch *search = mSearchs[i]; + if (search) + { + +#ifndef DEBUG_SEARCH + std::cerr << "ftFileSearch::search() SLOT: " << i; + std::cerr << std::endl; +#endif + if (search->search(hash, size, hintflags, info)) + { + return true; + } + } + } + /* found nothing */ + return false; } +bool ftSearchDummy::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const +{ +#ifndef DEBUG_SEARCH + std::cerr << "ftSearchDummy::search(" << hash << ", " << size; + std::cerr << ", " << hintflags << ");"; + std::cerr << std::endl; +#endif + return false; +} diff --git a/libretroshare/src/ft/ftfilesearch.h b/libretroshare/src/ft/ftfilesearch.h index a7dea20d9..24d7b8fb5 100644 --- a/libretroshare/src/ft/ftfilesearch.h +++ b/libretroshare/src/ft/ftfilesearch.h @@ -29,36 +29,31 @@ /* * ftFileSearch * - * This is actually implements the ftSearch Interface. + * This is implements an array of ftSearch Interfaces. * */ -#include "ft/ftsearch.h" +#include -class CacheStrapper; -class ftExtraList; -class FileIndexMonitor; -class FileIndexStore; +#include "ft/ftsearch.h" class ftFileSearch: public ftSearch { public: - ftFileSearch(CacheStrapper *c, ftExtraList *x, FileIndexMonitor *m, FileIndexStore *s) - :mCacheStrapper(c), mExtraList(x), mFileMonitor(m), mFileStore(s) { return; } + ftFileSearch(); -virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info); +bool addSearchMode(ftSearch *search, uint32_t hintflags); +virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; private: - CacheStrapper *mCacheStrapper; - ftExtraList *mExtraList; - FileIndexMonitor *mFileMonitor; - FileIndexStore *mFileStore; + // should have a mutex to protect vector.... + // but not really necessary as it is const most of the time. + std::vector mSearchs; }; - - +#endif diff --git a/libretroshare/src/ft/ftsearch.h b/libretroshare/src/ft/ftsearch.h index 576d54847..cb819ffd4 100644 --- a/libretroshare/src/ft/ftsearch.h +++ b/libretroshare/src/ft/ftsearch.h @@ -43,20 +43,18 @@ class ftSearch ftSearch() { return; } virtual ~ftSearch() { return; } -virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) = 0; +virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const = 0; }; -class ftSearchDummy +class ftSearchDummy: public ftSearch { + public: + ftSearchDummy() { return; } virtual ~ftSearchDummy() { return; } -virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) -{ - return false; -} - +virtual bool search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const; }; #endif diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index f3e196290..e1fe56e61 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -23,16 +23,56 @@ * */ +#include "util/rsdebug.h" +const int ftserverzone = 29539; + +#include "ft/ftserver.h" +#include "ft/ftextralist.h" +#include "ft/ftfilesearch.h" +#include "ft/ftcontroller.h" +#include "ft/ftdatamultiplex.h" + +#include "dbase/cachestrapper.h" +#include "dbase/fimonitor.h" +#include "dbase/fistore.h" + +#include "pqi/pqi.h" +#include "pqi/p3connmgr.h" + +#include +#include + /* Setup */ ftServer::ftServer(CacheStrapper *cStrapper, p3ConnectMgr *connMgr) :mCacheStrapper(cStrapper), mConnMgr(connMgr) { + +} + +void ftServer::setConfigDirectory(std::string path) +{ + mConfigPath = path; +} + +void ftServer::setPQInterface(PQInterface *pqi) +{ + +} + + /* Control Interface */ + + /* add Config Items (Extra, Controller) */ +void ftServer::addConfigComponents(p3ConfigMgr *mgr) +{ + /* NOT SURE ABOUT THIS ONE */ +} + /* Final Setup (once everything is assigned) */ -ftServer::SetupFtServer() +void ftServer::SetupFtServer() { /* make Controller */ - mFtController = new ftController(); + mFtController = new ftController(config_dir); NotifyBase *cb = getNotify(); /* setup FiStore/Monitor */ @@ -49,43 +89,42 @@ ftServer::SetupFtServer() /* extras List */ mFtExtra = new ftExtraList(); - mFtSearch = new ftFileSearch(mCacheStrapper, mFtExtra, mFiMon, mFiStore); + + mFtSearch = new ftFileSearch(); + //mFtSearch->addSearchMode(mCacheStrapper, RS_FILE_HINTS_CACHE); + mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA); + //mFtSearch->addSearchMode(mFiMon, RS_FILE_HINTS_LOCAL); + //mFtSearch->addSearchMode(mFiStore, RS_FILE_HINTS_REMOTE); mFtController -> setFtSearch(mFtSearch); - ftFiler -> setSaveBasePath(save_dir); + mFtController -> setSaveBasePath(save_dir); + + mFtDataplex = ftDataMultiplex(this, mFtSearch); return; } - /* Assign important variables */ -void setConfigDirectory(std::string path); - -void setPQInterface(PQInterface *pqi); - - /* Final Setup (once everything is assigned) */ -void SetupFtServer(); - - /* add Config Items (Extra, Controller) */ -void addConfigComponents(p3ConfigMgr *mgr); - - -void ftServer::setConfigDirectory(std::string path) -{ - mConfigPath = path; -} - -void ftServer::setPQInterface(PQInterface *pqi) - - /* Control Interface */ - -}; - void ftServer::StartupThreads() { - /* start up Controller thread */ + /* start up order - important for dependencies */ + + /* self contained threads */ + /* startup ExtraList Thread */ + mFtExtra->start(); + + /* startup Monitor Thread */ + /* startup the FileMonitor (after cache load) */ + mFiMon->setPeriod(600); /* 10 minutes */ + /* start it up */ + //mFiMon->setSharedDirectories(dbase_dirs); + mFiMon->start(); /* start own thread */ + //start(); + + /* Controller thread */ + mFtController->start(); } CacheStrapper *ftServer::getCacheStrapper() @@ -98,6 +137,7 @@ CacheTransfer *ftServer::getCacheTransfer() return mFtController; } + /***************************************************************/ /********************** RsFiles Interface **********************/ /***************************************************************/ @@ -113,23 +153,23 @@ bool ftServer::FileRequest(std::string fname, std::string hash, return mFtController->FileRequest(fname, hash, size, dest, flags); } -bool ftServer::FileCancel(std::string hash); +bool ftServer::FileCancel(std::string hash) { return mFtController->FileCancel(hash); } -bool ftServer::FileControl(std::string hash, uint32_t flags); +bool ftServer::FileControl(std::string hash, uint32_t flags) { return mFtController->FileControl(hash, flags); } -bool ftServer::FileClearCompleted(); +bool ftServer::FileClearCompleted() { return mFtController->FileClearCompleted(); } /* get Details of File Transfers */ -bool ftServer::FileDownloads(std::list &hashs); +bool ftServer::FileDownloads(std::list &hashs) { return mFtController->FileDownloads(hashs); } @@ -137,7 +177,7 @@ bool ftServer::FileDownloads(std::list &hashs); /* Directory Handling */ void ftServer::setDownloadDirectory(std::string path) { - return mFtController->setDownloadDirectory(path); + mFtController->setDownloadDirectory(path); } std::string ftServer::getDownloadDirectory() @@ -145,12 +185,12 @@ std::string ftServer::getDownloadDirectory() return mFtController->getDownloadDirectory(); } -void ftServer::setPartialsDirectory(std::string path); +void ftServer::setPartialsDirectory(std::string path) { - return mFtController->setPartialsDirectory(path); + mFtController->setPartialsDirectory(path); } -void ftServer::getPartialsDirectory() +std::string ftServer::getPartialsDirectory() { return mFtController->getPartialsDirectory(); } @@ -160,21 +200,21 @@ void ftServer::getPartialsDirectory() /************************* Other Access ************************/ /***************************************************************/ -bool ftServer::FileUploads(std::list &hashs); +bool ftServer::FileUploads(std::list &hashs) { - return mFtUploader->FileUploads(hashes); + return mFtDataplex->FileUploads(hashes); } -bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info); +bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) { bool found = false; - if (hintflags | DOWNLOADING) + if (hintflags | RS_FILE_HINTS_DOWNLOAD) { found = mFtController->FileDetails(hash, info); } - else if (hintflags | UPLOADING) + else if (hintflags | RS_FILE_HINTS_UPLOAD) { - found = mFtUploader->FileDetails(hash, info); + found = mFtDataplex->FileDetails(hash, info); } if (!found) @@ -194,17 +234,17 @@ bool ftServer::ExtraFileAdd(std::string fname, std::string hash, uint32_t size, return mFtExtra->addExtraFile(fname, hash, size, period, flags); } -bool ftServer::ExtraFileRemove(std::string hash, uin32_t flags); +bool ftServer::ExtraFileRemove(std::string hash, uint32_t flags) { return mFtExtra->removeExtraFile(hash, flags); } -bool ftServer::ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags); +bool ftServer::ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags) { return mFtExtra->hashExtraFile(localpath, period, flags); } -bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info); +bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info) { return mFtExtra->hashExtraFileDone(localpath, info); } @@ -213,12 +253,12 @@ bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info); /******************** Directory Listing ************************/ /***************************************************************/ -int ftServer::RequestDirDetails(std::string uid, std::string path, DirDetails &details); +int ftServer::RequestDirDetails(std::string uid, std::string path, DirDetails &details) { return mFiStore->RequestDirDetails(uid, path, details); } -int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags); +int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags) { return mFiStore->RequestDirDetails(ref, details, flags); } @@ -228,12 +268,12 @@ int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags); /***************************************************************/ -int ftServer::SearchKeywords(std::list keywords, std::list &results); +int ftServer::SearchKeywords(std::list keywords, std::list &results) { return mFiStore->SearchKeywords(keywords, results); } -int ftServer::SearchBoolExp(Expression * exp, std::list &results); +int ftServer::SearchBoolExp(Expression * exp, std::list &results) { return mFiStore->searchBoolExp(exp, results); } @@ -256,22 +296,22 @@ void ftServer::ForceDirectoryCheck() bool ftServer::InDirectoryCheck() { - return mFtMon->inDirectoryCheck(); + return mFiMon->inDirectoryCheck(); } bool ftServer::getSharedDirectories(std::list &dirs) { - return mFtMon->getSharedDirectories(dirs); + return mFiMon->getSharedDirectories(dirs); } bool ftServer::addSharedDirectory(std::string dir) { - return mFtMon->addSharedDirectory(dir); + return mFiMon->addSharedDirectory(dir); } bool ftServer::removeSharedDirectory(std::string dir) { - return mFtMon->removeSharedDirectory(dir); + return mFiMon->removeSharedDirectory(dir); } @@ -286,16 +326,309 @@ bool ftServer::removeSharedDirectory(std::string dir) protected: /* Key Functions to be overloaded for Full Configuration */ -virtual RsSerialiser *setupSerialiser(); -virtual std::list saveList(bool &cleanup); -virtual bool loadList(std::list load); +RsSerialiser *ftServer::setupSerialiser() +{ + return NULL; +} - private: -bool loadConfigMap(std::map &configMap); - /******************* p3 Config Overload ************************/ +std::list ftServer::saveList(bool &cleanup) +{ + std::list list; + return list; +} -}; +bool ftServer::loadList(std::list load) +{ + return true; +} + +bool ftServer::loadConfigMap(std::map &configMap) +{ + return true; +} + /***************************************************************/ + /********************** Data Flow **********************/ + /***************************************************************/ + /* Client Send */ +bool ftServer::sendDataRequest(std::string peerId, std::string hash, + uint64_t size, uint64_t offset, uint32_t chunksize) +{ + /* create a packet */ + /* push to networking part */ + RsFileRequest *rfi = new RsFileRequest(); + + /* id */ + rfi->PeerId(peerId); + + /* file info */ + rfi->file.filesize = size; + rfi->file.hash = hash; /* ftr->hash; */ + + /* offsets */ + rfi->fileoffset = offset; /* ftr->offset; */ + rfi->chunksize = chunksize; /* ftr->chunk; */ + + mP3iface->SendFileRequest(rfi); +} + +const uint32_t MAX_FT_CHUNK = 32 * 1024; /* 32K */ + + /* Server Send */ +bool ftServer::sendData(std::string peerId, std::string hash, uint64_t size, + uint64_t baseoffset, uint32_t chunksize, void *data) +{ + /* create a packet */ + /* push to networking part */ + uint32_t tosend = chunksize; + uint64_t offset = 0; + uint32_t chunk; + + + while(tosend > 0) + { + /* workout size */ + chunk = MAX_FT_CHUNK; + if (chunk > tosend) + { + chunk = tosend; + } + + /******** New Serialiser Type *******/ + + RsFileData *rfd = new RsFileData(); + + /* set id */ + rfd->PeerId(peerId); + + /* file info */ + rfd->fd.file.filesize = size; + rfd->fd.file.hash = hash; + rfd->fd.file.name = ""; /* blank other data */ + rfd->fd.file.path = ""; + rfd->fd.file.pop = 0; + rfd->fd.file.age = 0; + + rfd->fd.file_offset = baseoffset + offset; + + /* file data */ + rfd->fd.binData.setBinData( + &(((uint8_t *) data)[offset]), chunk); + + mP3iface->SendFileData(rfd); + + offset += chunk; + tosend -= chunk; + } + + /* clean up data */ + free(data); +} + + +int ftServer::tick() +{ + rslog(RSL_DEBUG_BASIC, ftserverzone, + "filedexserver::tick()"); + + /* the new Cache Hack() */ + FileStoreTick(); + + if (mP3iface == NULL) + { + std::ostringstream out; + rslog(RSL_DEBUG_BASIC, ftserverzone, + "filedexserver::tick() Invalid Interface()"); + + return 1; + } + + int moreToTick = 0; + + if (0 < mP3iface -> tick()) + { + moreToTick = 1; +#ifdef DEBUG_TICK + std::cerr << "filedexserver::tick() moreToTick from mP3iface" << std::endl; #endif + } + + if (0 < handleInputQueues()) + { + moreToTick = 1; +#ifdef DEBUG_TICK + std::cerr << "filedexserver::tick() moreToTick from InputQueues" << std::endl; +#endif + } + return moreToTick; +} + + +// This function needs to be divided up. +bool ftServer::handleInputQueues() +{ + handleCacheData(); + handleFileData(); +} + +bool ftServer::handleCacheData() +{ + // get all the incoming results.. and print to the screen. + RsCacheRequest *cr; + RsCacheItem *ci; + + // Loop through Search Results. + int i = 0; + int i_init = 0; + + //std::cerr << "filedexserver::handleInputQueues()" << std::endl; + while((ci = mP3iface -> GetSearchResult()) != NULL) + { + //std::cerr << "filedexserver::handleInputQueues() Recvd SearchResult (CacheResponse!)" << std::endl; + std::ostringstream out; + if (i++ == i_init) + { + out << "Recieved Search Results:" << std::endl; + } + ci -> print(out); + rslog(RSL_DEBUG_BASIC, ftserverzone, out.str()); + + /* these go to the CacheStrapper! */ + CacheData data; + data.cid = CacheId(ci->cacheType, ci->cacheSubId); + data.hash = ci->file.hash; + data.size = ci->file.filesize; + data.name = ci->file.name; + data.path = ci->file.path; + data.pid = ci->PeerId(); + data.pname = mAuthMgr->getName(ci->PeerId()); + mCacheStrapper->recvCacheResponse(data, time(NULL)); + + delete ci; + } + + // now requested Searches. + i_init = i; + while((cr = mP3iface -> RequestedSearch()) != NULL) + { + /* just delete these */ + std::ostringstream out; + out << "Requested Search:" << std::endl; + cr -> print(out); + rslog(RSL_DEBUG_BASIC, ftserverzone, out.str()); + delete cr; + } + + + // Now handle it replacement (pushed cache results) + { + std::list > cacheUpdates; + std::list >::iterator it; + + mCacheStrapper->getCacheUpdates(cacheUpdates); + for(it = cacheUpdates.begin(); it != cacheUpdates.end(); it++) + { + /* construct reply */ + RsCacheItem *ci = new RsCacheItem(); + + /* id from incoming */ + ci -> PeerId(it->first); + + ci -> file.hash = (it->second).hash; + ci -> file.name = (it->second).name; + ci -> file.path = ""; // (it->second).path; + ci -> file.filesize = (it->second).size; + ci -> cacheType = (it->second).cid.type; + ci -> cacheSubId = (it->second).cid.subid; + +#ifdef SERVER_DEBUG + std::ostringstream out2; + out2 << "Outgoing CacheStrapper Update -> RsCacheItem:" << std::endl; + ci -> print(out2); + std::cerr << out2.str() << std::endl; +#endif + + //rslog(RSL_DEBUG_BASIC, ftserverzone, out2.str()); + mP3iface -> SendSearchResult(ci); + } + } +} + + +bool ftServer::handleFileData() +{ + // now File Input. + RsFileRequest *fr; + RsFileData *fd; + + int i_init = 0; + int i = 0; + + i_init = i; + while((fr = mP3iface -> GetFileRequest()) != NULL ) + { +#ifdef SERVER_DEBUG + std::cerr << "filedexserver::handleInputQueues() Recvd ftFiler Request" << std::endl; + std::ostringstream out; + if (i == i_init) + { + out << "Incoming(Net) File Item:" << std::endl; + } + fr -> print(out); + rslog(RSL_DEBUG_BASIC, ftserverzone, out.str()); +#endif + + i++; /* count */ + mFtDataplex->recvDataRequest(fr->PeerId(), + fr->file.hash, fr->file.filesize, + fr->fileoffset, fr->chunksize); + +FileInfo(ffr); + delete fr; + + } + + // now File Data. + i_init = i; + while((fd = mP3iface -> GetFileData()) != NULL ) + { +#ifdef SERVER_DEBUG + //std::cerr << "filedexserver::handleInputQueues() Recvd ftFiler Data" << std::endl; + std::ostringstream out; + if (i == i_init) + { + out << "Incoming(Net) File Data:" << std::endl; + } + fd -> print(out); + rslog(RSL_DEBUG_BASIC, ftserverzone, out.str()); +#endif + i++; /* count */ + + /* incoming data */ + mFtDataplex->recvData(fd->PeerId(), + fd->fd.file.hash, fd->fd.file.filesize, + fd->fd.file_offset, + fd->fd.binData.bin_len, + fd->fd.binData.bin_data); + + /* we've stolen the data part -> so blank before delete + */ + fd->fd.TlvShallowClear(); + delete fd; + } + + if (i > 0) + { + return 1; + } + return 0; +} + +/********************************** + ********************************** + ********************************** + *********************************/ + + diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 9e1950a8b..4ddf74225 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -43,9 +43,11 @@ #include #include +#include "ft/ftdata.h" #include "rsiface/rsfiles.h" #include "dbase/cachestrapper.h" +#include "pqi/pqi.h" #include "pqi/p3cfgmgr.h" @@ -58,8 +60,11 @@ class FileIndexMonitor; class ftController; class ftExtraList; +class ftFileSearch; -class ftServer: public RsFiles +class ftDataMultiplex; + +class ftServer: public RsFiles, public ftDataSend { public: @@ -68,58 +73,75 @@ class ftServer: public RsFiles /******************** Setup ************************************/ /***************************************************************/ - ftServer(CacheStrapper *cStrapper) { return; } + ftServer(CacheStrapper *cStrapper, p3ConnectMgr *connMgr); /* Assign important variables */ void setConfigDirectory(std::string path); void setPQInterface(PQInterface *pqi); - /* Final Setup (once everything is assigned) */ -void SetupFtServer(); /* add Config Items (Extra, Controller) */ void addConfigComponents(p3ConfigMgr *mgr); +CacheStrapper *getCacheStrapper(); CacheTransfer *getCacheTransfer(); + /* Final Setup (once everything is assigned) */ +void SetupFtServer(); +void StartupThreads(); + /***************************************************************/ /*************** Control Interface *****************************/ /************** (Implements RsFiles) ***************************/ /***************************************************************/ +/*** + * Control of Downloads + ***/ virtual bool FileRequest(std::string fname, std::string hash, - uint32_t size, std::string dest, uint32_t flags); + uint32_t size, std::string dest, uint32_t flags); virtual bool FileCancel(std::string hash); virtual bool FileControl(std::string hash, uint32_t flags); virtual bool FileClearCompleted(); - /* get Details of File Transfers */ +/*** + * Download/Upload Details + ***/ virtual bool FileDownloads(std::list &hashs); virtual bool FileUploads(std::list &hashs); virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info); - /* Access ftExtraList - Details */ +/*** + * Extra List Access + ***/ virtual bool ExtraFileAdd(std::string fname, std::string hash, uint32_t size, - uint32_t period, uint32_t flags); + uint32_t period, uint32_t flags); virtual bool ExtraFileRemove(std::string hash, uint32_t flags); -virtual bool ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags); +virtual bool ExtraFileHash(std::string localpath, + uint32_t period, uint32_t flags); virtual bool ExtraFileStatus(std::string localpath, FileInfo &info); - /* Directory Listing */ + +/*** + * Directory Listing / Search Interface + ***/ virtual int RequestDirDetails(std::string uid, std::string path, DirDetails &details); virtual int RequestDirDetails(void *ref, DirDetails &details, uint32_t flags); - /* Search Interface */ virtual int SearchKeywords(std::list keywords, std::list &results); virtual int SearchBoolExp(Expression * exp, std::list &results); - /* Utility Functions */ +/*** + * Utility Functions + ***/ virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath); virtual void ForceDirectoryCheck(); virtual bool InDirectoryCheck(); - /* Directory Handling */ +/*** + * Directory Handling + ***/ virtual void setDownloadDirectory(std::string path); virtual void setPartialsDirectory(std::string path); virtual std::string getDownloadDirectory(); @@ -129,8 +151,6 @@ virtual bool getSharedDirectories(std::list &dirs); virtual bool addSharedDirectory(std::string dir); virtual bool removeSharedDirectory(std::string dir); -virtual int reScanDirs(); -virtual int check_dBUpdate(); /***************************************************************/ /*************** Control Interface *****************************/ @@ -139,9 +159,20 @@ virtual int check_dBUpdate(); /***************************************************************/ /*************** Data Transfer Interface ***********************/ /***************************************************************/ +public: +virtual bool sendData(std::string peerId, std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize, void *data); +virtual bool sendDataRequest(std::string peerId, + std::string hash, uint64_t size, + uint64_t offset, uint32_t chunksize); -bool requestData(std::string peerId, std::string hash, - uint64_t offset, uint32_t size); + /*************** Internal Transfer Fns *************************/ +virtual int tick(); + +private: +bool handleInputQueues(); +bool handleCacheData(); +bool handleFileData(); /******************* p3 Config Overload ************************/ protected: @@ -155,18 +186,34 @@ bool loadConfigMap(std::map &configMap); /******************* p3 Config Overload ************************/ /*************************** p3 Config Overload ********************/ + + private: + + /**** INTERNAL FUNCTIONS ***/ +virtual int reScanDirs(); +virtual int check_dBUpdate(); + private: /* no need for Mutex protection - * as each component is protected independently. */ + P3Interface *mP3iface; /* XXX THIS NEEDS PROTECTION */ + p3AuthMgr *mAuthMgr; + p3ConnectMgr *mConnMgr; + CacheStrapper *mCacheStrapper; + ftController *mFtController; ftExtraList *mFtExtra; - FileIndexStore *fiStore; - FileIndexMonitor *fimon; + ftDataMultiplex *mFtDataplex; + + FileIndexStore *mFiStore; + FileIndexMonitor *mFiMon; + + ftFileSearch *mFtSearch; RsMutex srvMutex; std::string mConfigPath; diff --git a/libretroshare/src/ft/fttransfermodule.cc b/libretroshare/src/ft/fttransfermodule.cc index 8dfd0e07d..446221317 100644 --- a/libretroshare/src/ft/fttransfermodule.cc +++ b/libretroshare/src/ft/fttransfermodule.cc @@ -21,21 +21,29 @@ * * Please report all bugs and problems to "retroshare@lunamutt.com". * - */ - -#include "fttransfermodule.h" - -const uint32_t PQIPEER_OFFLINE_CHECK = 120; /* check every 2 minutes */ -const uint32_t PQIPEER_DOWNLOAD_TIMEOUT = 60; /* time it out, -> offline after 60 secs */ -const uint32_t PQIPEER_DOWNLOAD_CHECK = 10; /* desired delta = 10 secs */ -const uint32_t PQIPEER_DOWNLOAD_TOO_FAST = 8; /* 8 secs */ -const uint32_t PQIPEER_DOWNLOAD_TOO_SLOW = 12; /* 12 secs */ -const uint32_t PQIPEER_DOWNLOAD_MIN_DELTA = 5; /* 5 secs */ - -ftTransferModule::ftTransferModule(ftFileCreator *fc,ftClientModule *cm) - :mFileCreator(fc),mClientModule(cm),mFlag(0) -{} - + */ + +#include "fttransfermodule.h" + +const uint32_t PQIPEER_OFFLINE_CHECK = 120; /* check every 2 minutes */ +const uint32_t PQIPEER_DOWNLOAD_TIMEOUT = 60; /* time it out, -> offline after 60 secs */ +const uint32_t PQIPEER_DOWNLOAD_CHECK = 10; /* desired delta = 10 secs */ +const uint32_t PQIPEER_DOWNLOAD_TOO_FAST = 8; /* 8 secs */ +const uint32_t PQIPEER_DOWNLOAD_TOO_SLOW = 12; /* 12 secs */ +const uint32_t PQIPEER_DOWNLOAD_MIN_DELTA = 5; /* 5 secs */ + +ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm) + :mFileCreator(fc), mMultiplexor(dm), mFlag(0) +{ + mHash = mFileCreator->getHash(); + mSize = mFileCreator->getFileSize(); + + // Dummy for Testing (should be handled independantly for + // each peer. + mChunkSize = 10000; + return; +} + ftTransferModule::~ftTransferModule() {} @@ -50,12 +58,12 @@ bool ftTransferModule::setFileSources(std::list peerIds) return 1; } -bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t maxRate); //state = ONLINE/OFFLINE +bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t maxRate) //state = ONLINE/OFFLINE { bool found = false; std::list::iterator it; - it = peerIds.begin(); - while (( it != peerIds.end())&&(!found)) + it = mFileSources.begin(); + while (( it != mFileSources.end())&&(!found)) { if ((*it) == peerId) found = true; @@ -66,12 +74,12 @@ bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t m std::map::iterator mit; mit = mOnlinePeers.find(peerId); - if (mit == map::end) + if (mit == mOnlinePeers.end()) { peerInfo pInfo; pInfo.state = state; pInfo.desiredRate = maxRate; - mOnlinePeers.insert(pair(peerId,pInfo)); + mOnlinePeers[peerId] = pInfo; } else { @@ -86,26 +94,42 @@ uint32_t ftTransferModule::getDataRate(std::string peerId) { std::map::iterator mit; mit = mOnlinePeers.find(peerId); - if (mit == map::end) + if (mit == mOnlinePeers.end()) return 0; else return (mit->second).actualRate; } -void ftTransferModule::requestData(uint64_t offset, uint32_t chunk_size) -{ - mClientModule->requestData(mHash,offset,chunk_size); -} - + + //interface to client module +bool ftTransferModule::recvFileData(std::string peerId, uint64_t offset, + uint32_t chunk_size, void *data) +{ +#ifdef FT_DEBUG + std::cerr << "ftTransferModule::recvFileData()"; + std::cerr << " peerId: " << peerId; + std::cerr << " offset: " << offset; + std::cerr << " chunksize: " << chunk_size; + std::cerr << std::endl; +#endif + + return storeData(offset, chunk_size, data); +} + +void ftTransferModule::requestData(std::string peerId, uint64_t offset, uint32_t chunk_size) +{ + mMultiplexor->sendDataRequest(peerId, mHash, mSize, offset,chunk_size); +} + bool ftTransferModule::getChunk(uint64_t &offset, uint32_t &chunk_size) { mFileCreator->getMissingChunk(offset, chunk_size); } bool ftTransferModule::storeData(uint64_t offset, uint32_t chunk_size,void *data) -{ - mFileCreator->storeData(offset, chunk_size, data); -} +{ + mFileCreator -> addFileData(offset, chunk_size, data); +} void ftTransferModule::queryInactive() { @@ -133,7 +157,7 @@ void ftTransferModule::queryInactive() (mit->second).req_size = mChunkSize; (mit->second).lastTS = ts; (mit->second).state = PQIPEER_DOWNLOADING; - requestData(mOffset,mChunkSize); + requestData(mit->first, mOffset,mChunkSize); } else mFlag = 1; } @@ -142,7 +166,7 @@ void ftTransferModule::queryInactive() //file request has been sent to peer side, but no response received yet case PQIPEER_DOWNLOADING: if (ts - ((mit->second).lastTS) > PQIPEER_DOWNLOAD_CHECK) - requestData((mit->second).req_loc,(mit->second).req_size); //give a push + requestData(mit->first, (mit->second).req_loc,(mit->second).req_size); //give a push break; //file response has been received or peer side is just ready for download @@ -154,7 +178,7 @@ void ftTransferModule::queryInactive() (mit->second).req_size = mChunkSize; (mit->second).lastTS = ts; (mit->second).state = PQIPEER_DOWNLOADING; - requestData(mOffset,mChunkSize); + requestData(mit->first, mOffset,mChunkSize); } else mFlag = 1; break; @@ -192,7 +216,7 @@ bool ftTransferModule::resumeTransfer() return 1; } -void ftTransferModule::completeFileTransfer() +bool ftTransferModule::completeFileTransfer() { } @@ -205,3 +229,11 @@ int ftTransferModule::tick() return 0; } + + +void ftTransferModule::adjustSpeed() +{ + return; +} + + diff --git a/libretroshare/src/ft/fttransfermodule.h b/libretroshare/src/ft/fttransfermodule.h index d8a9d51f6..e9467660c 100644 --- a/libretroshare/src/ft/fttransfermodule.h +++ b/libretroshare/src/ft/fttransfermodule.h @@ -34,11 +34,16 @@ * It must be able to cope with varying data rates and dropped peers without flooding the system with too many requests. * */ - -#include -#include -#include - + +#include +#include +#include + +#include "ft/ftfilecreator.h" +#include "ft/ftdatamultiplex.h" + +#include "util/rsthreads.h" + const int PQIPEER_INIT = 0x0000; const int PQIPEER_NOT_ONLINE = 0x0001; const int PQIPEER_DOWNLOADING = 0x0002; @@ -47,13 +52,15 @@ const int PQIPEER_SUSPEND = 0x0010; class Request { + public: uint64_t offset; uint32_t chunkSize; }; class peerInfo { - std:string peerId; + public: + std::string peerId; uint32_t state; uint32_t desiredRate; Request lastRequest; @@ -66,50 +73,55 @@ class peerInfo time_t lastTS; }; -class ftTransferModule -{ -public: - ftTransferModule(ftFileCreator *fc,ftClientModule *cm); - ~ftTransferModule(); - - //interface to download controller - bool setFileSources(std::list peerIds); - bool setPeerState(std::string peerId,uint32_t state,uint32_t maxRate); //state = ONLINE/OFFLINE - uint32_t getDataRate(std::string peerId); - bool stopTransfer(); - bool resumeTransfer(); - bool completeFileTransfer(); - - //interface to client module - bool recvFileData(uint64_t offset, uint32_t chunk_size, void *data); //called by ftClientModule when data arrives - void requestData(uint64_t offset, uint32_t chunk_size); - - //interface to file creator , just wrapper functions that call the ftFileCreator - bool getChunk(uint64_t &offset, uint32_t &chunk_size); - bool storeData(uint64_t offset, uint32_t chunk_size, void *data); - - //internal used functions - void queryInactive(); - - int tick(); - - /* add by DrBob for interfaces */ - std::string hash() { return mHash; } - uint64_t size() { return mSize; } - -public: - ftFileCreator *mFileCreator; - ftClientModule *mClientModule; - -private: - std::string mHash; - uint64_t mSize; - std::list mFileSources; - std::map mOnlinePeers; - - uint64_t mOffset; - uint32_t mChunkSize; - bool mFlag; //1:transfer complete, 0: not complete -}; - -#endif //FT_TRANSFER_MODULE_HEADER +class ftTransferModule +{ +public: + ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm); + ~ftTransferModule(); + + //interface to download controller + bool setFileSources(std::list peerIds); + bool setPeerState(std::string peerId,uint32_t state,uint32_t maxRate); //state = ONLINE/OFFLINE + uint32_t getDataRate(std::string peerId); + bool stopTransfer(); + bool resumeTransfer(); + bool completeFileTransfer(); + + //interface to multiplex module + bool recvFileData(std::string peerId, uint64_t offset, + uint32_t chunk_size, void *data); + void requestData(std::string peerId, uint64_t offset, uint32_t chunk_size); + + //interface to file creator + bool getChunk(uint64_t &offset, uint32_t &chunk_size); + bool storeData(uint64_t offset, uint32_t chunk_size, void *data); + + int tick(); + + std::string hash() { return mHash; } + uint64_t size() { return mSize; } + + //internal used functions + void queryInactive(); + void adjustSpeed(); + +private: + + /* These have independent Mutexes / are const locally (no Mutex protection)*/ + ftFileCreator *mFileCreator; + ftDataMultiplex *mMultiplexor; + + std::string mHash; + uint64_t mSize; + + RsMutex tfMtx; /* below is mutex protected */ + + std::list mFileSources; + std::map mOnlinePeers; + + uint64_t mOffset; + uint32_t mChunkSize; + bool mFlag; //1:transfer complete, 0: not complete +}; + +#endif //FT_TRANSFER_MODULE_HEADER diff --git a/libretroshare/src/rsiface/rsfiles.h b/libretroshare/src/rsiface/rsfiles.h index 936f2e4de..0daca7d35 100644 --- a/libretroshare/src/rsiface/rsfiles.h +++ b/libretroshare/src/rsiface/rsfiles.h @@ -51,19 +51,22 @@ const uint32_t RS_FILE_CTRL_STREAM_VIDEO = 0x0006; /************************************ * Used To indicate where to search. + * + * The Order of these is very important, + * it specifies the search order too. + * */ -const uint32_t RS_FILE_HINTS_MASK = 0x00ff; +const uint32_t RS_FILE_HINTS_MASK = 0x00ffffff; -const uint32_t RS_FILE_HINTS_CACHE = 0x0001; -const uint32_t RS_FILE_HINTS_EXTRA = 0x0002; -const uint32_t RS_FILE_HINTS_LOCAL = 0x0004; -const uint32_t RS_FILE_HINTS_REMOTE = 0x0008; -const uint32_t RS_FILE_HINTS_DOWNLOAD = 0x0010; -const uint32_t RS_FILE_HINTS_UPLOAD = 0x0020; - -const uint32_t RS_FILE_HINTS_SPEC_ONLY = 0x1000; +const uint32_t RS_FILE_HINTS_CACHE = 0x00000001; +const uint32_t RS_FILE_HINTS_EXTRA = 0x00000002; +const uint32_t RS_FILE_HINTS_LOCAL = 0x00000004; +const uint32_t RS_FILE_HINTS_REMOTE = 0x00000008; +const uint32_t RS_FILE_HINTS_DOWNLOAD = 0x00000010; +const uint32_t RS_FILE_HINTS_UPLOAD = 0x00000020; +const uint32_t RS_FILE_HINTS_SPEC_ONLY = 0x01000000; const uint32_t RS_FILE_EXTRA_DELETE = 0x0010; @@ -79,43 +82,27 @@ virtual ~RsFiles() { return; } /****************************************/ /* download */ -/* Required Interfaces ...... - * - * 1) Access to downloading / uploading files. - */ - -/* get Details of File Transfers */ -virtual bool FileDownloads(std::list &hashs) = 0; -virtual bool FileUploads(std::list &hashs) = 0; -virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0; - - -/* - * 2) Control of Downloads. - * - */ - +/*** + * Control of Downloads. + ***/ virtual bool FileRequest(std::string fname, std::string hash, uint32_t size, std::string dest, uint32_t flags) = 0; virtual bool FileCancel(std::string hash) = 0; virtual bool FileControl(std::string hash, uint32_t flags) = 0; virtual bool FileClearCompleted() = 0; +/*** + * Download / Upload Details. + ***/ +virtual bool FileDownloads(std::list &hashs) = 0; +virtual bool FileUploads(std::list &hashs) = 0; +virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0; -/* - * 3) Addition of Extra Files... From File System - * These are Hashed and stored in the 'Hidden Files' section - * which can only be accessed if you know the hash. - * - * FileHash is called to start the hashing process, - * and add the file to the HiddenStore. - * - * FileHashStatus is called to lookup files - * and see if the hashing is completed. - */ -/* Access ftExtraList - Details */ +/*** + * Extra List Access + ***/ virtual bool ExtraFileAdd(std::string fname, std::string hash, uint32_t size, uint32_t period, uint32_t flags) = 0; virtual bool ExtraFileRemove(std::string hash, uint32_t flags) = 0; @@ -124,25 +111,25 @@ virtual bool ExtraFileHash(std::string localpath, virtual bool ExtraFileStatus(std::string localpath, FileInfo &info) = 0; -/* - * 4) Search and Listing Interface +/*** + * Directory Listing / Search Interface */ - -/* Directory Listing / Search Interface */ virtual int RequestDirDetails(std::string uid, std::string path, DirDetails &details) = 0; virtual int RequestDirDetails(void *ref, DirDetails &details, uint32_t flags) = 0; virtual int SearchKeywords(std::list keywords, std::list &results) = 0; virtual int SearchBoolExp(Expression * exp, std::list &results) = 0; -/* - * 5) Directory Control / Shared Files Utility Functions. - */ - +/*** + * Utility Functions. + ***/ virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath) = 0; virtual void ForceDirectoryCheck() = 0; virtual bool InDirectoryCheck() = 0; +/*** + * Directory Control + ***/ virtual void setDownloadDirectory(std::string path) = 0; virtual void setPartialsDirectory(std::string path) = 0; virtual std::string getDownloadDirectory() = 0; diff --git a/libretroshare/src/serialiser/rstlvtypes.cc b/libretroshare/src/serialiser/rstlvtypes.cc index 58ab162fc..98f9729e3 100644 --- a/libretroshare/src/serialiser/rstlvtypes.cc +++ b/libretroshare/src/serialiser/rstlvtypes.cc @@ -33,6 +33,10 @@ #include #include +void RsTlvItem::TlvShallowClear() +{ + TlvClear(); /* unless overloaded! */ +} std::ostream &RsTlvItem::printBase(std::ostream &out, std::string clsName, uint16_t indent) { @@ -102,6 +106,11 @@ void RsTlvBinaryData::TlvClear() { free(bin_data); } + TlvShallowClear(); +} + +void RsTlvBinaryData::TlvShallowClear() +{ bin_data = NULL; bin_len = 0; } diff --git a/libretroshare/src/serialiser/rstlvtypes.h b/libretroshare/src/serialiser/rstlvtypes.h index aa58fc9a1..c1337b2ab 100644 --- a/libretroshare/src/serialiser/rstlvtypes.h +++ b/libretroshare/src/serialiser/rstlvtypes.h @@ -47,6 +47,7 @@ class RsTlvItem virtual ~RsTlvItem() { return; } virtual uint16_t TlvSize() = 0; virtual void TlvClear() = 0; +virtual void TlvShallowClear(); /*! Don't delete allocated data */ virtual bool SetTlv(void *data, uint32_t size, uint32_t *offset) = 0; /* serialise */ virtual bool GetTlv(void *data, uint32_t size, uint32_t *offset) = 0; /* deserialise */ virtual std::ostream &print(std::ostream &out, uint16_t indent) = 0; @@ -66,6 +67,8 @@ class RsTlvBinaryData: public RsTlvItem virtual ~RsTlvBinaryData(); virtual uint16_t TlvSize(); virtual void TlvClear(); /*! Initialize fields to empty legal values ( "0", "", etc) */ +virtual void TlvShallowClear(); /*! Don't delete the binary data */ + /// Serialise. /*! Serialise Tlv to buffer(*data) of 'size' bytes starting at *offset */ virtual bool SetTlv(void *data, uint32_t size, uint32_t *offset); diff --git a/libretroshare/src/util/rsthreads.h b/libretroshare/src/util/rsthreads.h index 0ffc0e5b9..2a613ab77 100644 --- a/libretroshare/src/util/rsthreads.h +++ b/libretroshare/src/util/rsthreads.h @@ -76,13 +76,17 @@ virtual void run() = 0; /* called once the thread is started */ }; -class RsQueueThread +class RsQueueThread: public RsThread { + public: + RsQueueThread(uint32_t min, uint32_t max, double relaxFactor ); virtual ~RsQueueThread() { return; } virtual void run(); + protected: + virtual bool workQueued() = 0; virtual bool doWork() = 0;