More Improvements to FileTransfer:

* Added TlvShallowClear() to serialisers
 * Added RsQueueThread for periodic queue processes.
 * Completed ftDataMultiplex which replaces ftServer/ClientModules.
 * Added Server Queue / Thread to ftDataMultiplex.
 * Added ftdataplextest to exercise ftDataMultiplex.
 * Generalised ftFileSearch to handle an array of ftSearch classes.
 * Tweaked rsfiles.h #defines to match new ftFileSearch scheme.
 * Added Generic ftData Interfaces for Testing.
 * added ftDataSend/Recv Interfaces to ftServer + ftDataMultiplex respectively.
 * Completed much of ftServer (External Interface), but not yet done.
 * Extra debugging and small changes to ftExtraList
 * Integrated new ftTransferModule with minor interface changes.



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@660 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/ft/Makefile

@@ -7,12 +7,13 @@ RS_TOP_DIR = ..
 include $(RS_TOP_DIR)/scripts/config.mk
 ###############################################################
 
-RSOBJ =  ftfileprovider.o ftfilecreator.o ftextralist.o ftdatamultiplex.o 
+RSOBJ =  ftdata.o ftfileprovider.o ftfilecreator.o ftextralist.o ftdatamultiplex.o ftfilesearch.o fttransfermodule.o 
-#ftservermodule.o
 
-TESTOBJ  = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o
+#ftserver.o
 
-TESTS  = ftfileprovidertest ftfilecreatortest ftextralisttest
+TESTOBJ  = ftfileprovidertest.o ftfilecreatortest.o ftextralisttest.o ftdataplextest.o
+
+TESTS  = ftfileprovidertest ftfilecreatortest ftextralisttest ftdataplextest
 
 all: librs tests
 
@@ -25,6 +26,9 @@ ftfileprovidertest : ftfileprovidertest.o
 ftextralisttest : ftextralisttest.o 
 	$(CC) $(CFLAGS) -o ftextralisttest ftextralisttest.o $(LIBS)
 
+ftdataplextest : ftdataplextest.o 
+	$(CC) $(CFLAGS) -o ftdataplextest ftdataplextest.o $(LIBS)
+
 ###############################################################
 include $(RS_TOP_DIR)/scripts/rules.mk
 ###############################################################

libretroshare/src/ft/ftcontroller.h

@@ -38,6 +38,10 @@
  *
  */
 
+class ftFileCreator;
+class ftTransferModule;
+class ftFileProvider;
+
 class ftController: public CacheTransfer, public RsThread, public pqiMonitor, public p3Config
 {
 	public:
@@ -96,8 +100,8 @@ virtual bool    loadList(std::list<RsItem *> load);
 
 	RsMutex ctrlMutex;
 
-	std::list<FileDetails> incomingQueue;
+	std::list<FileInfo> incomingQueue;
-	std::map<std::string, FileDetails> mCompleted;
+	std::map<std::string, FileInfo> mCompleted;
 
 	std::map<std::string, ftTransferModule *> mTransfers;
 	std::map<std::string, ftFileCreator *> mFileCreators;

libretroshare/src/ft/ftdata.cc

@@ -0,0 +1,82 @@
+/*
+ * libretroshare/src/ft: ftdata.cc
+ *
+ * File Transfer for RetroShare.
+ *
+ * Copyright 2008 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "ft/ftdata.h"
+
+/******* Pair of Send/Recv (Only need to handle Send side) ******/
+ftDataSendPair::ftDataSendPair(ftDataRecv *recv)
+	:mDataRecv(recv) 
+{ 
+	return; 
+}
+
+	/* Client Send */
+bool	ftDataSendPair::sendDataRequest(std::string peerId, std::string hash, 
+			uint64_t size, uint64_t offset, uint32_t chunksize)
+{
+	return mDataRecv->recvDataRequest(peerId,hash,size,offset,chunksize);
+}
+
+	/* Server Send */
+bool	ftDataSendPair::sendData(std::string peerId, 
+			std::string hash, uint64_t size,
+                        uint64_t offset, uint32_t chunksize, void *data)
+{
+	return mDataRecv->recvData(peerId, hash,size,offset,chunksize,data);
+}
+
+
+	/* Client Send */
+bool	ftDataSendDummy::sendDataRequest(std::string peerId, std::string hash, 
+			uint64_t size, uint64_t offset, uint32_t chunksize)
+{
+	return true;
+}
+
+	/* Server Send */
+bool	ftDataSendDummy::sendData(std::string peerId, 
+			std::string hash, uint64_t size,
+                        uint64_t offset, uint32_t chunksize, void *data)
+{
+	return true;
+}
+
+
+	/* Client Recv */
+bool	ftDataRecvDummy::recvData(std::string peerId, 
+			std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize, void *data)
+{
+	return true;
+}
+
+
+	/* Server Recv */
+bool	ftDataRecvDummy::recvDataRequest(std::string peerId, std::string hash, 
+			uint64_t size, uint64_t offset, uint32_t chunksize)
+{
+	return true;
+}
+

libretroshare/src/ft/ftdata.h

@@ -46,12 +46,12 @@ class ftDataSend
 virtual ~ftDataSend() { return; }
 
 	/* Client Send */
-virtual bool	sendDataRequest(std::string peerId, std::string hash, 
+virtual bool    sendDataRequest(std::string peerId, std::string hash, 
-			uint64_t offset, uint32_t size) = 0;
+			uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
 
 	/* Server Send */
-virtual bool	sendData(std::string peerId, std::string hash, 
+virtual bool    sendData(std::string peerId, std::string hash, uint64_t size,
-			uint64_t offset, uint32_t size, void *data) = 0;
+                        uint64_t offset, uint32_t chunksize, void *data) = 0;
 
 };
 
@@ -66,12 +66,64 @@ class ftDataRecv
 virtual ~ftDataRecv() { return; }
 
 	/* Client Recv */
-virtual bool	recvData(std::string peerId, std::string hash, 
+virtual bool    recvData(std::string peerId, std::string hash, uint64_t size, 
-			uint64_t offset, uint32_t size, void *data) = 0;
+			uint64_t offset, uint32_t chunksize, void *data) = 0;
 
 	/* Server Recv */
-virtual bool	recvDataRequest(std::string peerId, std::string hash, 
+virtual bool    recvDataRequest(std::string peerId, std::string hash, 
-			uint64_t offset, uint32_t size) = 0;
+			uint64_t size, uint64_t offset, uint32_t chunksize) = 0;
+
+
+};
+
+/******* Pair of Send/Recv (Only need to handle Send side) ******/
+class ftDataSendPair: public ftDataSend
+{
+	public:
+
+	ftDataSendPair(ftDataRecv *recv);
+virtual ~ftDataSendPair() { return; }
+
+	/* Client Send */
+virtual bool    sendDataRequest(std::string peerId, std::string hash, 
+			uint64_t size, uint64_t offset, uint32_t chunksize);
+
+	/* Server Send */
+virtual bool    sendData(std::string peerId, std::string hash, uint64_t size,
+                        uint64_t offset, uint32_t chunksize, void *data);
+
+	ftDataRecv *mDataRecv;
+};
+
+
+class ftDataSendDummy: public ftDataSend
+{
+	public:
+virtual ~ftDataSendDummy() { return; }
+
+	/* Client Send */
+virtual bool    sendDataRequest(std::string peerId, std::string hash, 
+			uint64_t size, uint64_t offset, uint32_t chunksize);
+
+	/* Server Send */
+virtual bool    sendData(std::string peerId, std::string hash, uint64_t size,
+                        uint64_t offset, uint32_t chunksize, void *data);
+
+};
+
+class ftDataRecvDummy: public ftDataRecv
+{
+	public:
+
+virtual ~ftDataRecvDummy() { return; }
+
+	/* Client Recv */
+virtual bool    recvData(std::string peerId, std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize, void *data);
+
+	/* Server Recv */
+virtual bool    recvDataRequest(std::string peerId, std::string hash, 
+			uint64_t size, uint64_t offset, uint32_t chunksize);
 
 
 };

libretroshare/src/ft/ftdatamultiplex.cc

@@ -35,6 +35,11 @@
 #include "ft/ftfileprovider.h"
 #include "ft/ftsearch.h"
 
+/* For Thread Behaviour */
+const uint32_t DMULTIPLEX_MIN	= 10; /* 1ms sleep */
+const uint32_t DMULTIPLEX_MAX   = 1000; /* 1 sec sleep */
+const double   DMULTIPLEX_RELAX = 0.5; /* ??? */
+ 
 ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
 	:mModule(module), mCreator(creator)
 {
@@ -44,15 +49,16 @@ ftClient::ftClient(ftTransferModule *module, ftFileCreator *creator)
 const uint32_t FT_DATA		= 0x0001;
 const uint32_t FT_DATA_REQ	= 0x0002;
 
-ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t offset, uint32_t chunk, void *data)
+ftRequest::ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data)
-	:mType(type), mPeerId(peerId), mHash(hash),
+	:mType(type), mPeerId(peerId), mHash(hash), mSize(size),
 	mOffset(offset), mChunk(chunk), mData(data)
 {
 	return;
 }
 
 ftDataMultiplex::ftDataMultiplex(ftDataSend *server, ftSearch *search)
-	:mDataSend(server), mSearch(search)
+	:RsQueueThread(DMULTIPLEX_MIN, DMULTIPLEX_MAX, DMULTIPLEX_RELAX),
+	mDataSend(server),  mSearch(search)
 {
 	return;
 }
@@ -90,49 +96,135 @@ bool	ftDataMultiplex::removeTransferModule(ftTransferModule *mod, ftFileCreator
 	/*************** SEND INTERFACE (calls ftDataSend) *******************/
 
 	/* Client Send */
-bool	ftDataMultiplex::sendDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size)
+bool	ftDataMultiplex::sendDataRequest(std::string peerId, 
+	std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize)
 {
-	return mDataSend->sendDataRequest(peerId, hash, offset, size);
+	return mDataSend->sendDataRequest(peerId,hash,size,offset,chunksize);
 }
 
 	/* Server Send */
-bool	ftDataMultiplex::sendData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data)
+bool	ftDataMultiplex::sendData(std::string peerId, 
+		std::string hash, uint64_t size, 
+		uint64_t offset, uint32_t chunksize, void *data)
 {
-	return mDataSend->sendData(peerId, hash, offset, size, data);
+	return mDataSend->sendData(peerId,hash,size,offset,chunksize,data);
 }
 
 
 	/*************** RECV INTERFACE (provides ftDataRecv) ****************/
 
 	/* Client Recv */
-bool	ftDataMultiplex::recvData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data)
+bool	ftDataMultiplex::recvData(std::string peerId, 
+	std::string hash, uint64_t size, 
+	uint64_t offset, uint32_t chunksize, void *data)
 {
 	/* Store in Queue */
 	RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
 	mRequestQueue.push_back(
-		ftRequest(FT_DATA, peerId, hash, offset, size, data));
+		ftRequest(FT_DATA,peerId,hash,size,offset,chunksize,data));
 
 	return true;
 }
 
 
 	/* Server Recv */
-bool	ftDataMultiplex::recvDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size)
+bool	ftDataMultiplex::recvDataRequest(std::string peerId, 
+		std::string hash, uint64_t size, 
+		uint64_t offset, uint32_t chunksize)
 {
 	/* Store in Queue */
 	RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
 	mRequestQueue.push_back(
-		ftRequest(FT_DATA_REQ, peerId, hash, offset, size, NULL));
+		ftRequest(FT_DATA_REQ,peerId,hash,size,offset,chunksize,NULL));
 
 	return true;
 }
 
 
 /*********** BACKGROUND THREAD OPERATIONS ***********/
+bool 	ftDataMultiplex::workQueued()
+{
+	RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
+	if (mRequestQueue.size() > 0)
+	{
+		return true;
+	}
+
+	if (mSearchQueue.size() > 0)
+	{
+		return true;
+	}
+
+	return false;
+}
+	
+bool 	ftDataMultiplex::doWork()
+{
+	bool doRequests = true;
+
+	/* Handle All the current Requests */		
+	while(doRequests)
+	{
+		ftRequest req;
+
+		{
+			RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
+			if (mRequestQueue.size() == 0)
+			{
+				doRequests = false;
+				continue;
+			}
+
+			req = mRequestQueue.front();
+			mRequestQueue.pop_front();
+		}
+
+		/* MUTEX FREE */
+
+		switch(req.mType)
+		{
+		  case FT_DATA:
+			handleRecvData(req.mPeerId, req.mHash, req.mSize,
+				req.mOffset, req.mChunk, req.mData);
+			break;
+
+		  case FT_DATA_REQ:
+			handleRecvDataRequest(req.mPeerId, req.mHash,
+				req.mSize,  req.mOffset, req.mChunk);
+			break;
+
+		  default:
+			break;
+		}
+	}
+
+	/* Only Handle One Search Per Period.... 
+   	 * Lower Priority
+	 */
+	ftRequest req;
+
+	{
+		RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
+		if (mSearchQueue.size() == 0)
+		{
+			/* Finished */
+			return true;
+		}
+
+		req = mSearchQueue.front();
+		mSearchQueue.pop_front();
+	}
+
+	handleSearchRequest(req.mPeerId, req.mHash, req.mSize, 
+					req.mOffset, req.mChunk);
+
+	return true;
+}
 
 
 bool	ftDataMultiplex::handleRecvData(std::string peerId, 
-		std::string hash, uint64_t offset, uint32_t size, void *data)
+			std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize, void *data)
 {
 	RsStackMutex stack(dataMtx); /******* LOCK MUTEX ******/
 	std::map<std::string, ftClient>::iterator it;
@@ -142,7 +234,7 @@ bool	ftDataMultiplex::handleRecvData(std::string peerId,
 		return false;
 	}
 	
-	(it->second).mModule->recvFileData(peerId, offset, size, data);
+	(it->second).mModule->recvFileData(peerId, offset, chunksize, data);
 
 	return true;
 }
@@ -150,7 +242,8 @@ bool	ftDataMultiplex::handleRecvData(std::string peerId,
 
 	/* called by ftTransferModule */
 bool	ftDataMultiplex::handleRecvDataRequest(std::string peerId, 
-			std::string hash, uint64_t offset, uint32_t size)
+			std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize)
 {
 	/**** Find Files *****/
 
@@ -159,7 +252,7 @@ bool	ftDataMultiplex::handleRecvDataRequest(std::string peerId,
 	if (mClients.end() != (cit = mClients.find(hash)))
 	{
 		locked_handleServerRequest((cit->second).mCreator, 
-					peerId, hash, offset, size);
+					peerId, hash, size, offset, chunksize);
 		return true;
 	}
 	
@@ -167,26 +260,28 @@ bool	ftDataMultiplex::handleRecvDataRequest(std::string peerId,
 	if (mServers.end() != (sit = mServers.find(hash)))
 	{
 		locked_handleServerRequest(sit->second,
-					peerId, hash, offset, size);
+					peerId, hash, size, offset, chunksize);
 		return true;
 	}
 
 
 	/* Add to Search Queue */
 	mSearchQueue.push_back(
-		ftRequest(FT_DATA_REQ, peerId, hash, offset, size, NULL));
+		ftRequest(FT_DATA_REQ, peerId, hash, 
+				size, offset, chunksize, NULL));
 
 	return true;
 }
 
 bool	ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider,
-	std::string peerId, std::string hash, uint64_t offset, uint32_t size)
+		std::string peerId, std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize)
 {
 	void *data = malloc(size);
-	if (provider->getFileData(offset, size, data))
+	if (provider->getFileData(offset, chunksize, data))
 	{
 		/* send data out */
-		sendData(peerId, hash, offset, size, data);
+		sendData(peerId, hash, size, offset, chunksize, data);
 		return true;
 	}
 	return false;
@@ -194,7 +289,8 @@ bool	ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider,
 
 
 bool	ftDataMultiplex::handleSearchRequest(std::string peerId, 
-			std::string hash, uint64_t offset, uint32_t size)
+			std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize)
 {
 
 	{
@@ -235,7 +331,7 @@ bool	ftDataMultiplex::handleSearchRequest(std::string peerId,
 
 		/* handle request finally */
 		locked_handleServerRequest(provider,
-					peerId, hash, offset, size);
+					peerId, hash, size, offset, chunksize);
 		return true;
 	}
 	return false;

libretroshare/src/ft/ftdatamultiplex.h

@@ -61,11 +61,15 @@ class ftRequest
 {
 	public:
 
-	ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t offset, uint32_t chunk, void *data);
+	ftRequest(uint32_t type, std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunk, void *data);
+
+	ftRequest()
+	:mType(0), mSize(0), mOffset(0), mChunk(0), mData(NULL) { return; }
 
 	uint32_t mType;
 	std::string mPeerId;
 	std::string mHash;
+	uint64_t mSize;
 	uint64_t mOffset;
 	uint32_t mChunk;
 	void *mData;
@@ -73,7 +77,7 @@ class ftRequest
 
 	
 
-class ftDataMultiplex: public ftDataRecv
+class ftDataMultiplex: public ftDataRecv, public RsQueueThread
 {
 
 	public:
@@ -89,36 +93,48 @@ bool	removeTransferModule(ftTransferModule *mod, ftFileCreator *f);
 	/*************** SEND INTERFACE (calls ftDataSend) *******************/
 
 	/* Client Send */
-bool	sendDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size);
+bool	sendDataRequest(std::string peerId, std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize);
 
 	/* Server Send */
-bool	sendData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data);
+bool	sendData(std::string peerId, std::string hash, uint64_t size, 
+			uint64_t offset, uint32_t chunksize, void *data);
 
 
 	/*************** RECV INTERFACE (provides ftDataRecv) ****************/
 
 	/* Client Recv */
-virtual bool	recvData(std::string peerId, std::string hash, uint64_t offset, uint32_t size, void *data);
+virtual bool	recvData(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
 
 	/* Server Recv */
-virtual bool	recvDataRequest(std::string peerId, std::string hash, uint64_t offset, uint32_t size);
+virtual bool	recvDataRequest(std::string peerId, std::string hash, uint64_t size, uint64_t offset, uint32_t chunksize);
 
 
+	protected:
+
+	/* Overloaded from RsQueueThread */
+virtual bool workQueued();
+virtual bool doWork();
+
 	private:
 
 	/* Handling Job Queues */
 bool    handleRecvData(std::string peerId,
-                std::string hash, uint64_t offset, uint32_t size, void *data);
+                std::string hash, uint64_t size, 
+		uint64_t offset, uint32_t chunksize, void *data);
 
 bool    handleRecvDataRequest(std::string peerId,
-                        std::string hash, uint64_t offset, uint32_t size);
+                        std::string hash, uint64_t size,
+			uint64_t offset, uint32_t chunksize);
 
 bool    handleSearchRequest(std::string peerId,
-                        std::string hash, uint64_t offset, uint32_t size);
+                        std::string hash, uint64_t size,
+			uint64_t offset, uint32_t chunksize);
 
 	/* We end up doing the actual server job here */
 bool    locked_handleServerRequest(ftFileProvider *provider, 
-	std::string peerId, std::string hash, uint64_t offset, uint32_t size);
+	std::string peerId, std::string hash, uint64_t size, 
+	uint64_t offset, uint32_t chunksize);
 
 	RsMutex dataMtx;
 

libretroshare/src/ft/ftdataplextest.cc

@@ -0,0 +1,174 @@
+/*
+ * libretroshare/src/ft: ftextralisttest.cc
+ *
+ * File Transfer for RetroShare.
+ *
+ * Copyright 2008 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+/*
+ * Test for Multiplexor.....
+ * As this is a key middle component, it is hard to test without other bits.
+ * It relies on ftFileProvider/ftFileCreator/ftTransferModule... 
+ *
+ * And has dummy ftDataSend and ftSearch.
+ *
+ */
+
+#include "ft/ftextralist.h"
+#include "ft/ftdatamultiplex.h"
+#include "ft/ftfilesearch.h"
+
+void	do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list<std::string> &files);
+
+
+void usage(char *name)
+{
+	std::cerr << "Usage: " << name << " [-p <period>] [-d <dperiod>] <path> [<path2> ... ] ";
+	std::cerr << std::endl;
+}
+	
+int main(int argc, char **argv)
+{
+        int c;
+        uint32_t period = 1;
+        uint32_t dPeriod = 600; /* default 10 minutes */
+
+        std::list<std::string> fileList;
+
+        while(-1 != (c = getopt(argc, argv, "d:p:")))
+        {
+                switch (c)
+                {
+                case 'p':
+                        period = atoi(optarg);
+                        break;
+                case 'd':
+                        dPeriod = atoi(optarg);
+                        break;
+                default:
+                        usage(argv[0]);
+                        break;
+                }
+        }
+
+	if (optind >= argc)
+	{
+		std::cerr << "Missing Files" << std::endl;
+		usage(argv[0]);
+	}
+
+	for(; optind < argc; optind++)
+	{
+		std::cerr << "Adding: " << argv[optind] << std::endl;
+		fileList.push_back(std::string(argv[optind]));
+	}
+
+	ftExtraList *eList = new ftExtraList();
+	eList->start();
+
+	ftSearch *ftsd = new ftSearchDummy(); 
+	ftFileSearch *ftfs = new ftFileSearch();
+
+	ftfs-> addSearchMode(ftsd, RS_FILE_HINTS_LOCAL);
+	ftfs-> addSearchMode(eList, RS_FILE_HINTS_EXTRA);
+
+	ftDataSend *ftds = new ftDataSendDummy();
+
+	/* setup Actual Test bit */
+	ftDataMultiplex *ftmplex = new ftDataMultiplex(ftds, ftfs);
+	ftmplex->start();
+
+	/* Setup Search with some valid results */
+
+
+	/* Request Data */	
+
+	/* now work the thread */
+	std::list<std::string>::iterator it;
+	uint32_t flags = 0;
+	for(it = fileList.begin(); it  != fileList.end(); it++)
+	{
+		eList->hashExtraFile(*it, dPeriod, flags);
+	}
+
+
+	/* now request files from ftDataMultiplex */
+
+	/* just request random data packets first */
+	do_random_server_test(ftmplex, eList, fileList);
+}
+
+
+
+uint32_t do_random_server_iteration(ftDataMultiplex *mplex, ftExtraList *eList, std::list<std::string> &files)
+{
+	std::cerr << "do_random_server_iteration()";
+	std::cerr << std::endl;
+
+	std::list<std::string>::iterator it;
+	uint32_t i = 0;
+	for(it = files.begin(); it != files.end(); it++)
+	{
+		FileInfo info;
+		if (eList->hashExtraFileDone(*it, info))
+		{
+			std::cerr << "Hash Done for: " << *it;
+			std::cerr << std::endl;
+			std::cerr << info << std::endl;
+
+			std::cerr << "Requesting Data Packet";
+			std::cerr << std::endl;
+
+        		/* Server Recv */
+                	uint64_t offset = 10000;
+			uint32_t chunk = 20000;
+			mplex->recvDataRequest("Peer", info.hash, info.size, offset, chunk);
+			
+			i++;
+		}
+		else
+		{
+			std::cerr << "do_random_server_iteration() Hash Not Done for: " << *it;
+			std::cerr << std::endl;
+		}
+	}
+
+	return i;
+
+}
+
+
+void	do_random_server_test(ftDataMultiplex *mplex, ftExtraList *eList, std::list<std::string> &files)
+{
+	std::cerr << "do_random_server_test()";
+	std::cerr << std::endl;
+
+	uint32_t size = files.size();
+	while(size > do_random_server_iteration(mplex, eList, files))
+	{
+		std::cerr << "do_random_server_test() sleep";
+		std::cerr << std::endl;
+
+		sleep(10);
+	}
+}
+
+

libretroshare/src/ft/ftextralist.cc

@@ -26,6 +26,8 @@
 #include "ft/ftextralist.h"
 #include "util/rsdir.h"
 
+#define DEBUG_ELIST	1
+
 ftExtraList::ftExtraList()
 	:p3Config(CONFIG_FT_EXTRA_LIST)
 {
@@ -41,6 +43,11 @@ void ftExtraList::run()
 
 	while (1)
 	{
+#ifdef  DEBUG_ELIST
+		std::cerr << "ftExtraList::run() Iteration";
+		std::cerr << std::endl;
+#endif
+
 		now = time(NULL);
 
 		{
@@ -76,15 +83,29 @@ void ftExtraList::run()
 
 void ftExtraList::hashAFile()
 {
+#ifdef  DEBUG_ELIST
+	std::cerr << "ftExtraList::hashAFile()";
+	std::cerr << std::endl;
+#endif
+
 	/* extract entry from the queue */
 	FileDetails details;
 
 	{
 		RsStackMutex stack(extMutex);
+
+		if (mToHash.size() == 0)
+			return;
+
 		details = mToHash.front();
 		mToHash.pop_front();
 	}
 
+#ifdef  DEBUG_ELIST
+	std::cerr << "Hashing: " << details.info.path;
+	std::cerr << std::endl;
+#endif
+
 	/* hash it! */
 	std::string name, hash;
 	uint64_t size;
@@ -110,6 +131,16 @@ void ftExtraList::hashAFile()
 bool	ftExtraList::addExtraFile(std::string path, std::string hash, 
 				uint64_t size, uint32_t period, uint32_t flags)
 {
+#ifdef  DEBUG_ELIST
+	std::cerr << "ftExtraList::addExtraFile() path: " << path;
+	std::cerr << " hash: " << hash;
+	std::cerr << " size: " << size;
+	std::cerr << " period: " << period;
+	std::cerr << " flags: " << flags;
+
+	std::cerr << std::endl;
+#endif
+
 	RsStackMutex stack(extMutex);
 
 	FileDetails details;
@@ -126,11 +157,42 @@ bool	ftExtraList::addExtraFile(std::string path, std::string hash,
 	/* stick it in the available queue */
 	mFiles[details.info.hash] = details;
 
+	return true;
 }
 
+bool ftExtraList::removeExtraFile(std::string hash, uint32_t flags)
+{
+#ifdef  DEBUG_ELIST
+	std::cerr << "ftExtraList::removeExtraFile()";
+	std::cerr << " hash: " << hash;
+	std::cerr << " flags: " << flags;
+
+	std::cerr << std::endl;
+#endif
+
+	RsStackMutex stack(extMutex);
+
+	std::map<std::string, FileDetails>::iterator it;
+	it = mFiles.find(hash);
+	if (it == mFiles.end())
+	{
+		return false;
+	}
+
+	mFiles.erase(it);
+
+	return true;
+}
+
+
 	
 bool	ftExtraList::cleanupOldFiles()
 {
+#ifdef  DEBUG_ELIST
+	std::cerr << "ftExtraList::cleanupOldFiles()";
+	std::cerr << std::endl;
+#endif
+
 	RsStackMutex stack(extMutex);
 
 	time_t now = time(NULL);
@@ -159,6 +221,7 @@ bool	ftExtraList::cleanupOldFiles()
 			}
 		}
 	}
+	return true;
 }
 
 
@@ -169,6 +232,14 @@ bool	ftExtraList::cleanupOldFiles()
 
 bool 	ftExtraList::hashExtraFile(std::string path, uint32_t period, uint32_t flags)
 {
+#ifdef  DEBUG_ELIST
+	std::cerr << "ftExtraList::hashExtraFile() path: " << path;
+	std::cerr << " period: " << period;
+	std::cerr << " flags: " << flags;
+
+	std::cerr << std::endl;
+#endif
+
 	/* add into queue */
 	RsStackMutex stack(extMutex);
 
@@ -180,6 +251,11 @@ bool 	ftExtraList::hashExtraFile(std::string path, uint32_t period, uint32_t fla
 
 bool	ftExtraList::hashExtraFileDone(std::string path, FileInfo &info)
 {
+#ifdef  DEBUG_ELIST
+	std::cerr << "ftExtraList::hashExtraFileDone()";
+	std::cerr << std::endl;
+#endif
+
 	std::string hash;
 	{
 		/* Find in the path->hash map */
@@ -192,19 +268,25 @@ bool	ftExtraList::hashExtraFileDone(std::string path, FileInfo &info)
 		}
 		hash = it->second;
 	}
-	return searchExtraFiles(hash, info);
+	return search(hash, 0, 0, info);
 }
 
 	/***
 	 * Search Function - used by File Transfer 
 	 *
 	 **/
-bool	ftExtraList::searchExtraFiles(std::string hash, FileInfo &info)
+bool    ftExtraList::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
 {
+
+#ifdef  DEBUG_ELIST
+	std::cerr << "ftExtraList::search()";
+	std::cerr << std::endl;
+#endif
+
 	RsStackMutex stack(extMutex);
 
 	/* find hash */
-	std::map<std::string, FileDetails>::iterator fit;
+	std::map<std::string, FileDetails>::const_iterator fit;
 	if (mFiles.end() == (fit = mFiles.find(hash)))
 	{
 		return false;

libretroshare/src/ft/ftextralist.h

@@ -57,6 +57,7 @@
 #include <map>
 #include <string>
 
+#include "ft/ftsearch.h"
 #include "util/rsthreads.h"
 #include "rsiface/rsfiles.h"
 #include "pqi/p3cfgmgr.h"
@@ -108,7 +109,7 @@ const uint32_t CONFIG_FT_EXTRA_LIST 	= 1;
 const uint32_t CLEANUP_PERIOD		= 600; /* 10 minutes */
 
 
-class ftExtraList: public RsThread, public p3Config
+class ftExtraList: public RsThread, public p3Config, public ftSearch
 {
 
 	public:
@@ -122,6 +123,8 @@ class ftExtraList: public RsThread, public p3Config
 bool		addExtraFile(std::string path, std::string hash, 
 				uint64_t size, uint32_t period, uint32_t flags);
 
+bool		removeExtraFile(std::string hash, uint32_t flags);
+
 		/***
 		 * Hash file, and add to the files, 
 		 * file is removed after period.
@@ -132,9 +135,10 @@ bool	 	hashExtraFileDone(std::string path, FileInfo &info);
 
 		/***
 		 * Search Function - used by File Transfer 
+		 * implementation of ftSearch.
 		 *
 		 **/
-bool		searchExtraFiles(std::string hash, FileInfo &info);
+virtual bool    search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
 
 		/***
 		 * Thread Main Loop 
@@ -156,7 +160,7 @@ virtual bool    loadList(std::list<RsItem *> load);
 void	hashAFile();
 bool	cleanupOldFiles();
 
-		RsMutex extMutex;
+		mutable RsMutex extMutex;
 
 		std::list<FileDetails> mToHash;
 

libretroshare/src/ft/ftextralisttest.cc

@@ -142,7 +142,7 @@ void	displayExtraListDetails(ftExtraList *eList, std::list<std::string> toHash,
 	for(it = hashed.begin(); it != hashed.end(); it++)
 	{
 		FileInfo info;
-		if (eList->searchExtraFiles(*it, info))
+		if (eList->search(*it, 0, 0, info))
 		{
 			std::cerr << "displayExtraListDetails() Found Hash: " << *it;
 			std::cerr << std::endl;

libretroshare/src/ft/ftfileprovider.cc

@@ -11,6 +11,16 @@ ftFileProvider::~ftFileProvider(){
 	}
 }
 
+std::string ftFileProvider::getHash()
+{
+	return hash;
+}
+
+uint64_t ftFileProvider::getFileSize()
+{
+	return total_size;
+}
+
 bool ftFileProvider::getFileData(uint64_t offset, uint32_t chunk_size, void *data)
 {
         RsStackMutex stack(ftPMutex); /********** STACK LOCKED MTX ******/

libretroshare/src/ft/ftfilesearch.cc

@@ -23,57 +23,142 @@
  *
  */
 
-#ifndef FT_FILE_SEARCH_HEADERd
-#define FT_FILE_SEARCH_HEADER
-
-/* 
- * ftFileSearch
- *
- * This is actually implements the ftSearch Interface.
- *
- */
-
 #include "ft/ftfilesearch.h"
-#include "dbase/cachestrapper.h"
-#include "dbase/fimonitor.h"
-#include "dbase/fistore.h"
 
-bool	ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info)
+const uint32_t MAX_SEARCHS = 24; /* lower 24 bits of hint */
+
+ftFileSearch::ftFileSearch()
+	:mSearchs(MAX_SEARCHS)
 {
-
+	uint32_t i;
-#if 0
+	for(i = 0; i < MAX_SEARCHS; i++)
-	/* actual search depends on the hints */
-	if (hintflags | CACHE)
 	{
-		mCacheStrapper->..
+		mSearchs[i] = NULL;		
 	}
+}
 
-	if (hintflags | LOCAL)
+bool	ftFileSearch::addSearchMode(ftSearch *search, uint32_t hintflags)
-	{
+{
+	hintflags &= 0x00ffffff;
 
-	}
+#ifndef DEBUG_SEARCH
-
+	std::cerr << "ftSearchDummy::addSearchMode() : " << hintflags;
-	if (hintflags | EXTRA)
+	std::cerr << std::endl;
-	{
-
-
-	}
-
-	if (hintflags | REMOTE)
-	{
-
-	}
-
-	private:
-
-	CacheStrapper *mCacheStrapper;
-	ftExtraList *mExtraList;
-	FileIndexMonitor *mFileMonitor;
-	FileIndexStore   *mFileStore;
 #endif
 
+	uint32_t i;
+	for  (i = 0; i < MAX_SEARCHS; i++)
+	{
+		uint32_t hints = hintflags >> i;
+		if (hints & 0x0001)
+		{
+			/* has the flag */
+			mSearchs[i] = search;
+
+#ifndef DEBUG_SEARCH
+			std::cerr << "ftSearchDummy::addSearchMode() to slot ";
+			std::cerr << i;
+			std::cerr << std::endl;
+#endif
+
+			return true;
+		}
+	}
+
+#ifndef DEBUG_SEARCH
+	std::cerr << "ftSearchDummy::addSearchMode() Failed";
+	std::cerr << std::endl;
+#endif
+
+	return false;
+}
+
+bool	ftFileSearch::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
+{
+	uint32_t hints, i;
+
+#ifndef DEBUG_SEARCH
+	std::cerr << "ftFileSearch::search(" << hash << ", " << size; 
+	std::cerr << ", " << hintflags << ");";
+	std::cerr << std::endl;
+#endif
+	
+	for  (i = 0; i < MAX_SEARCHS; i++)
+	{
+		hints = hintflags >> i;
+		if (hints & 0x0001)
+		{
+			/* has the flag */
+			ftSearch *search = mSearchs[i];
+			if (search)
+			{
+#ifndef DEBUG_SEARCH
+				std::cerr << "ftFileSearch::search() SLOT: ";
+				std::cerr << i;
+				std::cerr << std::endl;
+#endif
+				if (search->search(hash, size, hintflags, info))
+				{
+					return true;
+				}
+			}
+		}
+	}
+
+	/* if we haven't found it by now! - check if SPEC_ONLY flag is set */
+	if (hintflags & RS_FILE_HINTS_SPEC_ONLY)
+	{
+#ifndef DEBUG_SEARCH
+		std::cerr << "ftFileSearch::search() SPEC_ONLY: Failed";
+		std::cerr << std::endl;
+#endif
+		return false;
+	}
+
+#ifndef DEBUG_SEARCH
+	std::cerr << "ftSearchDummy::search() Searching Others:";
+	std::cerr << std::endl;
+#endif
+
+	/* if we don't have the SPEC_ONLY flag, 
+	 * we check through all the others 
+	 */
+	for  (i = 0; i < MAX_SEARCHS; i++)
+	{
+		hints = hintflags >> i;
+		if (hints & 0x0001)
+		{
+			continue;
+		}
+
+		/* has the flag */
+		ftSearch *search = mSearchs[i];
+		if (search)
+		{
+
+#ifndef DEBUG_SEARCH
+			std::cerr << "ftFileSearch::search() SLOT: " << i;
+			std::cerr << std::endl;
+#endif
+			if (search->search(hash, size, hintflags, info))
+			{
+				return true;
+			}
+		}
+	}
+	/* found nothing */
+	return false;
 }
 
 
+bool 	ftSearchDummy::search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const
+{
+#ifndef DEBUG_SEARCH
+	std::cerr << "ftSearchDummy::search(" << hash << ", " << size; 
+	std::cerr << ", " << hintflags << ");";
+	std::cerr << std::endl;
+#endif
+        return false;
+}
 
 

libretroshare/src/ft/ftfilesearch.h

@@ -29,36 +29,31 @@
 /* 
  * ftFileSearch
  *
- * This is actually implements the ftSearch Interface.
+ * This is implements an array of ftSearch Interfaces.
  *
  */
 
-#include "ft/ftsearch.h"
+#include <vector>
 
-class CacheStrapper;
+#include "ft/ftsearch.h"
-class ftExtraList;
-class FileIndexMonitor;
-class FileIndexStore;
 
 class ftFileSearch: public ftSearch
 {
 
 	public:
 
-	ftFileSearch(CacheStrapper *c, ftExtraList *x, FileIndexMonitor *m, FileIndexStore *s)
+	ftFileSearch();
-	:mCacheStrapper(c), mExtraList(x), mFileMonitor(m), mFileStore(s) { return; }
 
-virtual bool	search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info);
+bool    addSearchMode(ftSearch *search, uint32_t hintflags);
+virtual bool    search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
 
 	private:
 
-	CacheStrapper *mCacheStrapper;
+	// should have a mutex to protect vector....
-	ftExtraList *mExtraList;
+	// but not really necessary as it is const most of the time.
-	FileIndexMonitor *mFileMonitor;
-	FileIndexStore   *mFileStore;
 
+	std::vector<ftSearch *> mSearchs;
 };
 
 
-
+#endif
-

libretroshare/src/ft/ftsearch.h

@@ -43,20 +43,18 @@ class ftSearch
 
 	ftSearch() { return; }
 virtual ~ftSearch() { return; }
-virtual bool	search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) = 0;
+virtual bool	search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const = 0;
 
 };
 
 
-class ftSearchDummy
+class ftSearchDummy: public ftSearch
 {
+	public:
+
 	ftSearchDummy() { return; }
 virtual ~ftSearchDummy() { return; }
-virtual bool	search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info)
+virtual bool	search(std::string hash, uint64_t size, uint32_t hintflags, FileInfo &info) const;
-{
-	return false;
-}
-
 };
 
 #endif

libretroshare/src/ft/ftserver.cc

@@ -23,16 +23,56 @@
  *
  */
 
+#include "util/rsdebug.h"
+const int ftserverzone = 29539;
+
+#include "ft/ftserver.h"
+#include "ft/ftextralist.h"
+#include "ft/ftfilesearch.h"
+#include "ft/ftcontroller.h"
+#include "ft/ftdatamultiplex.h"
+
+#include "dbase/cachestrapper.h"
+#include "dbase/fimonitor.h"
+#include "dbase/fistore.h"
+
+#include "pqi/pqi.h"
+#include "pqi/p3connmgr.h"
+
+#include <iostream>
+#include <sstream>
+
 	/* Setup */
 ftServer::ftServer(CacheStrapper *cStrapper, p3ConnectMgr *connMgr)
 	:mCacheStrapper(cStrapper), mConnMgr(connMgr)
 {
 
+
+}
+
+void	ftServer::setConfigDirectory(std::string path)
+{
+	mConfigPath = path;
+}
+
+void	ftServer::setPQInterface(PQInterface *pqi)
+{
+	
+}
+
+	/* Control Interface */
+
+	/* add Config Items (Extra, Controller) */
+void	ftServer::addConfigComponents(p3ConfigMgr *mgr)
+{
+	/* NOT SURE ABOUT THIS ONE */
+}
+
 	/* Final Setup (once everything is assigned) */
-ftServer::SetupFtServer()
+void ftServer::SetupFtServer()
 {
 	/* make Controller */
-	mFtController = new ftController();
+	mFtController = new ftController(config_dir);
 	NotifyBase *cb = getNotify();
 
 	/* setup FiStore/Monitor */
@@ -49,43 +89,42 @@ ftServer::SetupFtServer()
 
 	/* extras List */
 	mFtExtra = new ftExtraList();
-	mFtSearch = new ftFileSearch(mCacheStrapper, mFtExtra, mFiMon, mFiStore);
+
+	mFtSearch = new ftFileSearch();
+	//mFtSearch->addSearchMode(mCacheStrapper, RS_FILE_HINTS_CACHE);
+	mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA);
+	//mFtSearch->addSearchMode(mFiMon, RS_FILE_HINTS_LOCAL);
+	//mFtSearch->addSearchMode(mFiStore, RS_FILE_HINTS_REMOTE);
 
 	mFtController -> setFtSearch(mFtSearch);
-	ftFiler -> setSaveBasePath(save_dir);
+	mFtController -> setSaveBasePath(save_dir);
+
+        mFtDataplex = ftDataMultiplex(this, mFtSearch);
 	
 	return;
 }
 
-	/* Assign important variables */
-void	setConfigDirectory(std::string path);
-
-void	setPQInterface(PQInterface *pqi);
-
-	/* Final Setup (once everything is assigned) */
-void	SetupFtServer();
-
-	/* add Config Items (Extra, Controller) */
-void	addConfigComponents(p3ConfigMgr *mgr);
-
-
-void	ftServer::setConfigDirectory(std::string path)
-{
-	mConfigPath = path;
-}
-
-void	ftServer::setPQInterface(PQInterface *pqi)
-
-	/* Control Interface */
-
-};
-
 
 void    ftServer::StartupThreads()
 {
-	/* start up Controller thread */
+	/* start up order - important for dependencies */
+
+	/* self contained threads */
+	/* startup ExtraList Thread */
+	mFtExtra->start();
+
+	/* startup Monitor Thread */
+	/* startup the FileMonitor (after cache load) */
+	mFiMon->setPeriod(600); /* 10 minutes */
+	/* start it up */
+	//mFiMon->setSharedDirectories(dbase_dirs);
+	mFiMon->start();
 
 	/* start own thread */
+	//start();
+
+	/* Controller thread */
+	mFtController->start();
 }
 
 CacheStrapper *ftServer::getCacheStrapper()
@@ -98,6 +137,7 @@ CacheTransfer *ftServer::getCacheTransfer()
 	return mFtController;
 }
 
+
 	/***************************************************************/
 	/********************** RsFiles Interface **********************/
 	/***************************************************************/
@@ -113,23 +153,23 @@ bool ftServer::FileRequest(std::string fname, std::string hash,
 	return mFtController->FileRequest(fname, hash, size, dest, flags);
 }
 
-bool ftServer::FileCancel(std::string hash);
+bool ftServer::FileCancel(std::string hash)
 {
 	return mFtController->FileCancel(hash);
 }
 
-bool ftServer::FileControl(std::string hash, uint32_t flags);
+bool ftServer::FileControl(std::string hash, uint32_t flags)
 {
 	return mFtController->FileControl(hash, flags);
 }
 
-bool ftServer::FileClearCompleted();
+bool ftServer::FileClearCompleted()
 {
 	return mFtController->FileClearCompleted();
 }
 
 	/* get Details of File Transfers */
-bool ftServer::FileDownloads(std::list<std::string> &hashs);
+bool ftServer::FileDownloads(std::list<std::string> &hashs)
 {
 	return mFtController->FileDownloads(hashs);
 }
@@ -137,7 +177,7 @@ bool ftServer::FileDownloads(std::list<std::string> &hashs);
 	/* Directory Handling */
 void ftServer::setDownloadDirectory(std::string path)
 {
-	return mFtController->setDownloadDirectory(path);
+	mFtController->setDownloadDirectory(path);
 }
 
 std::string ftServer::getDownloadDirectory()
@@ -145,12 +185,12 @@ std::string ftServer::getDownloadDirectory()
 	return mFtController->getDownloadDirectory();
 }
 
-void ftServer::setPartialsDirectory(std::string path);
+void ftServer::setPartialsDirectory(std::string path)
 {
-	return mFtController->setPartialsDirectory(path);
+	mFtController->setPartialsDirectory(path);
 }
 
-void ftServer::getPartialsDirectory()
+std::string ftServer::getPartialsDirectory()
 {
 	return mFtController->getPartialsDirectory();
 }
@@ -160,21 +200,21 @@ void ftServer::getPartialsDirectory()
 	/************************* Other Access ************************/
 	/***************************************************************/
 
-bool ftServer::FileUploads(std::list<std::string> &hashs);
+bool ftServer::FileUploads(std::list<std::string> &hashs)
 {
-	return mFtUploader->FileUploads(hashes);
+	return mFtDataplex->FileUploads(hashes);
 }
 
-bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info);
+bool ftServer::FileDetails(std::string hash, uint32_t hintflags, FileInfo &info)
 {
 	bool found = false;
-	if (hintflags | DOWNLOADING)
+	if (hintflags | RS_FILE_HINTS_DOWNLOAD)
 	{
 		found = mFtController->FileDetails(hash, info);
 	}
-	else if (hintflags | UPLOADING)
+	else if (hintflags | RS_FILE_HINTS_UPLOAD)
 	{
-		found = mFtUploader->FileDetails(hash, info);
+		found = mFtDataplex->FileDetails(hash, info);
 	}
 
 	if (!found)
@@ -194,17 +234,17 @@ bool  ftServer::ExtraFileAdd(std::string fname, std::string hash, uint32_t size,
 	return mFtExtra->addExtraFile(fname, hash, size, period, flags);
 }
 
-bool ftServer::ExtraFileRemove(std::string hash, uin32_t flags);
+bool ftServer::ExtraFileRemove(std::string hash, uint32_t flags)
 {
 	return mFtExtra->removeExtraFile(hash, flags);
 }
 
-bool ftServer::ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags);
+bool ftServer::ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags)
 {
 	return mFtExtra->hashExtraFile(localpath, period, flags);
 }
 
-bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info);
+bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info)
 {
 	return mFtExtra->hashExtraFileDone(localpath, info);
 }
@@ -213,12 +253,12 @@ bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info);
 	/******************** Directory Listing ************************/
 	/***************************************************************/
 
-int ftServer::RequestDirDetails(std::string uid, std::string path, DirDetails &details);
+int ftServer::RequestDirDetails(std::string uid, std::string path, DirDetails &details)
 {
 	return mFiStore->RequestDirDetails(uid, path, details);
 }
 	
-int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags);
+int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags)
 {
 	return mFiStore->RequestDirDetails(ref, details, flags);
 }
@@ -228,12 +268,12 @@ int ftServer::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags);
 	/***************************************************************/
 
 
-int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results);
+int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results)
 {
 	return mFiStore->SearchKeywords(keywords, results);
 }
 	
-int ftServer::SearchBoolExp(Expression * exp, std::list<FileDetail> &results);
+int ftServer::SearchBoolExp(Expression * exp, std::list<FileDetail> &results)
 {
 	return mFiStore->searchBoolExp(exp, results);
 }
@@ -256,22 +296,22 @@ void    ftServer::ForceDirectoryCheck()
 	
 bool    ftServer::InDirectoryCheck()
 {
-	return mFtMon->inDirectoryCheck();
+	return mFiMon->inDirectoryCheck();
 }
 	
 bool	ftServer::getSharedDirectories(std::list<std::string> &dirs)
 {
-	return mFtMon->getSharedDirectories(dirs);
+	return mFiMon->getSharedDirectories(dirs);
 }
 
 bool 	ftServer::addSharedDirectory(std::string dir)
 {
-	return mFtMon->addSharedDirectory(dir);
+	return mFiMon->addSharedDirectory(dir);
 }
 
 bool 	ftServer::removeSharedDirectory(std::string dir)
 {
-	return mFtMon->removeSharedDirectory(dir);
+	return mFiMon->removeSharedDirectory(dir);
 }
 
 
@@ -286,16 +326,309 @@ bool 	ftServer::removeSharedDirectory(std::string dir)
 
 	protected:
         /* Key Functions to be overloaded for Full Configuration */
-virtual RsSerialiser *setupSerialiser();
+RsSerialiser *ftServer::setupSerialiser()
-virtual std::list<RsItem *> saveList(bool &cleanup);
+{
-virtual bool    loadList(std::list<RsItem *> load);
+	return NULL;
+}
 
-	private:
+std::list<RsItem *> ftServer::saveList(bool &cleanup)
-bool  loadConfigMap(std::map<std::string, std::string> &configMap);
+{
-        /******************* p3 Config Overload ************************/
+	std::list<RsItem *> list;
+	return list;
+}
 
-};
+bool    ftServer::loadList(std::list<RsItem *> load)
+{
+	return true;
+}
+
+bool  ftServer::loadConfigMap(std::map<std::string, std::string> &configMap)
+{
+	return true;
+}
 
 
+	/***************************************************************/
+	/**********************     Data Flow     **********************/
+	/***************************************************************/
 
+	/* Client Send */
+bool	ftServer::sendDataRequest(std::string peerId, std::string hash, 
+			uint64_t size, uint64_t offset, uint32_t chunksize)
+{
+	/* create a packet */
+	/* push to networking part */
+	RsFileRequest *rfi = new RsFileRequest();
+
+	/* id */
+	rfi->PeerId(peerId);
+
+	/* file info */
+	rfi->file.filesize   = size;
+	rfi->file.hash       = hash; /* ftr->hash; */
+
+	/* offsets */
+	rfi->fileoffset = offset; /* ftr->offset; */
+	rfi->chunksize  = chunksize; /* ftr->chunk; */
+
+	mP3iface->SendFileRequest(rfi);
+}
+
+const uint32_t	MAX_FT_CHUNK  = 32 * 1024; /* 32K */
+
+	/* Server Send */
+bool	ftServer::sendData(std::string peerId, std::string hash, uint64_t size,
+			uint64_t baseoffset, uint32_t chunksize, void *data)
+{
+	/* create a packet */
+	/* push to networking part */
+	uint32_t tosend = chunksize;
+	uint64_t offset = 0;
+	uint32_t chunk;
+
+
+	while(tosend > 0)
+	{
+		/* workout size */
+		chunk = MAX_FT_CHUNK;
+		if (chunk > tosend)
+		{
+			chunk = tosend;
+		}
+
+		/******** New Serialiser Type *******/
+
+		RsFileData *rfd = new RsFileData();
+
+		/* set id */
+		rfd->PeerId(peerId);
+
+		/* file info */
+		rfd->fd.file.filesize = size;
+		rfd->fd.file.hash     = hash;
+		rfd->fd.file.name     = ""; /* blank other data */
+		rfd->fd.file.path     = "";
+		rfd->fd.file.pop      = 0;
+		rfd->fd.file.age      = 0;
+
+		rfd->fd.file_offset = baseoffset + offset;
+
+		/* file data */
+		rfd->fd.binData.setBinData(
+			&(((uint8_t *) data)[offset]), chunk);
+
+		mP3iface->SendFileData(rfd);
+
+		offset += chunk;
+		tosend -= chunk;
+	}
+
+	/* clean up data */
+	free(data);
+}
+
+
+int	ftServer::tick()
+{
+	rslog(RSL_DEBUG_BASIC, ftserverzone, 
+		"filedexserver::tick()");
+
+	/* the new Cache Hack() */
+	FileStoreTick();
+
+	if (mP3iface == NULL)
+	{
+		std::ostringstream out;
+		rslog(RSL_DEBUG_BASIC, ftserverzone, 
+			"filedexserver::tick() Invalid Interface()");
+
+		return 1;
+	}
+
+	int moreToTick = 0;
+
+	if (0 < mP3iface -> tick())
+	{
+		moreToTick = 1;
+#ifdef DEBUG_TICK
+		std::cerr << "filedexserver::tick() moreToTick from mP3iface" << std::endl;
 #endif
+	}
+
+	if (0 < handleInputQueues())
+	{
+		moreToTick = 1;
+#ifdef DEBUG_TICK
+		std::cerr << "filedexserver::tick() moreToTick from InputQueues" << std::endl;
+#endif
+	}
+	return moreToTick;
+}
+
+
+// This function needs to be divided up.
+bool    ftServer::handleInputQueues()
+{
+	handleCacheData();
+	handleFileData();
+}
+
+bool     ftServer::handleCacheData()
+{
+	// get all the incoming results.. and print to the screen.
+	RsCacheRequest *cr;
+	RsCacheItem    *ci;
+
+	// Loop through Search Results.
+	int i = 0;
+	int i_init = 0;
+
+	//std::cerr << "filedexserver::handleInputQueues()" << std::endl;
+	while((ci = mP3iface -> GetSearchResult()) != NULL)
+	{
+		//std::cerr << "filedexserver::handleInputQueues() Recvd SearchResult (CacheResponse!)" << std::endl;
+		std::ostringstream out;
+		if (i++ == i_init)
+		{
+			out << "Recieved Search Results:" << std::endl;
+		}
+		ci -> print(out);
+		rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
+
+		/* these go to the CacheStrapper! */
+		CacheData data;
+		data.cid = CacheId(ci->cacheType, ci->cacheSubId);
+		data.hash = ci->file.hash;
+		data.size = ci->file.filesize;
+		data.name = ci->file.name;
+		data.path = ci->file.path;
+		data.pid = ci->PeerId();
+		data.pname = mAuthMgr->getName(ci->PeerId());
+		mCacheStrapper->recvCacheResponse(data, time(NULL));
+
+		delete ci;
+	}
+
+	// now requested Searches.
+	i_init = i;
+	while((cr = mP3iface -> RequestedSearch()) != NULL)
+	{
+		/* just delete these */
+		std::ostringstream out;
+		out << "Requested Search:" << std::endl;
+		cr -> print(out);
+		rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
+		delete cr;
+	}
+
+
+	// Now handle it replacement (pushed cache results) 
+	{
+		std::list<std::pair<RsPeerId, CacheData> > cacheUpdates;
+		std::list<std::pair<RsPeerId, CacheData> >::iterator it;
+
+		mCacheStrapper->getCacheUpdates(cacheUpdates);
+		for(it = cacheUpdates.begin(); it != cacheUpdates.end(); it++)
+		{
+			/* construct reply */
+			RsCacheItem *ci = new RsCacheItem();
+	
+			/* id from incoming */
+			ci -> PeerId(it->first);
+
+			ci -> file.hash = (it->second).hash;
+			ci -> file.name = (it->second).name;
+			ci -> file.path = ""; // (it->second).path;
+			ci -> file.filesize = (it->second).size;
+			ci -> cacheType  = (it->second).cid.type;
+			ci -> cacheSubId =  (it->second).cid.subid;
+
+#ifdef SERVER_DEBUG 
+			std::ostringstream out2;
+			out2 << "Outgoing CacheStrapper Update -> RsCacheItem:" << std::endl;
+			ci -> print(out2);
+			std::cerr << out2.str() << std::endl;
+#endif
+
+			//rslog(RSL_DEBUG_BASIC, ftserverzone, out2.str());
+			mP3iface -> SendSearchResult(ci);
+		}
+	}
+}
+
+
+bool    ftServer::handleFileData()
+{
+	// now File Input.
+	RsFileRequest *fr;
+	RsFileData *fd;
+
+	int i_init = 0;
+	int i = 0;
+
+	i_init = i;
+	while((fr = mP3iface -> GetFileRequest()) != NULL )
+	{
+#ifdef SERVER_DEBUG 
+		std::cerr << "filedexserver::handleInputQueues() Recvd ftFiler Request" << std::endl;
+		std::ostringstream out;
+		if (i == i_init)
+		{
+			out << "Incoming(Net) File Item:" << std::endl;
+		}
+		fr -> print(out);
+		rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
+#endif
+
+		i++; /* count */
+		mFtDataplex->recvDataRequest(fr->PeerId(), 
+				fr->file.hash,  fr->file.filesize, 
+				fr->fileoffset, fr->chunksize);
+
+FileInfo(ffr);
+		delete fr;
+
+	}
+
+	// now File Data.
+	i_init = i;
+	while((fd = mP3iface -> GetFileData()) != NULL )
+	{
+#ifdef SERVER_DEBUG 
+		//std::cerr << "filedexserver::handleInputQueues() Recvd ftFiler Data" << std::endl;
+		std::ostringstream out;
+		if (i == i_init)
+		{
+			out << "Incoming(Net) File Data:" << std::endl;
+		}
+		fd -> print(out);
+		rslog(RSL_DEBUG_BASIC, ftserverzone, out.str());
+#endif
+		i++; /* count */
+
+		/* incoming data */
+		mFtDataplex->recvData(fd->PeerId(),
+				 fd->fd.file.hash,  fd->fd.file.filesize, 
+				fd->fd.file_offset, 
+				fd->fd.binData.bin_len, 
+				fd->fd.binData.bin_data);
+
+		/* we've stolen the data part -> so blank before delete 
+		 */
+		fd->fd.TlvShallowClear();
+		delete fd;
+	}
+
+	if (i > 0)
+	{
+		return 1;
+	}
+	return 0;
+}
+
+/**********************************
+ **********************************
+ **********************************
+ *********************************/
+
+

libretroshare/src/ft/ftserver.h

@@ -43,9 +43,11 @@
 #include <list>
 #include <iostream>
 
+#include "ft/ftdata.h"
 #include "rsiface/rsfiles.h"
 #include "dbase/cachestrapper.h"
 
+#include "pqi/pqi.h"
 #include "pqi/p3cfgmgr.h"
 
 
@@ -58,8 +60,11 @@ class FileIndexMonitor;
 
 class ftController;
 class ftExtraList;
+class ftFileSearch;
 
-class ftServer: public RsFiles
+class ftDataMultiplex;
+
+class ftServer: public RsFiles, public ftDataSend
 {
 
 	public:
@@ -68,58 +73,75 @@ class ftServer: public RsFiles
 	/******************** Setup ************************************/
 	/***************************************************************/
 
-	ftServer(CacheStrapper *cStrapper) { return; }
+	ftServer(CacheStrapper *cStrapper, p3ConnectMgr *connMgr);
 
 	/* Assign important variables */
 void	setConfigDirectory(std::string path);
 
 void	setPQInterface(PQInterface *pqi);
 
-	/* Final Setup (once everything is assigned) */
-void	SetupFtServer();
 
 	/* add Config Items (Extra, Controller) */
 void	addConfigComponents(p3ConfigMgr *mgr);
 
+CacheStrapper *getCacheStrapper();
 CacheTransfer *getCacheTransfer();
 
+	/* Final Setup (once everything is assigned) */
+void	SetupFtServer();
+void	StartupThreads();
+
 	/***************************************************************/
 	/*************** Control Interface *****************************/
 	/************** (Implements RsFiles) ***************************/
 	/***************************************************************/
 
+/***
+ * Control of Downloads
+ ***/
 virtual bool FileRequest(std::string fname, std::string hash, 
-				uint32_t size, std::string dest, uint32_t flags);
+			uint32_t size, std::string dest, uint32_t flags);
 virtual bool FileCancel(std::string hash);
 virtual bool FileControl(std::string hash, uint32_t flags);
 virtual bool FileClearCompleted();
 
-	/* get Details of File Transfers */
+/***
+ * Download/Upload Details 
+ ***/
 virtual bool FileDownloads(std::list<std::string> &hashs);
 virtual bool FileUploads(std::list<std::string> &hashs);
 virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info);
 
-	/* Access ftExtraList - Details */
+/***
+ * Extra List Access
+ ***/
 virtual bool ExtraFileAdd(std::string fname, std::string hash, uint32_t size, 
-						uint32_t period, uint32_t flags);
+			uint32_t period, uint32_t flags);
 virtual bool ExtraFileRemove(std::string hash, uint32_t flags);
-virtual bool ExtraFileHash(std::string localpath, uint32_t period, uint32_t flags);
+virtual bool ExtraFileHash(std::string localpath, 
+			uint32_t period, uint32_t flags);
 virtual bool ExtraFileStatus(std::string localpath, FileInfo &info);
 
-	/* Directory Listing */
+
+/***
+ * Directory Listing / Search Interface
+ ***/
 virtual int RequestDirDetails(std::string uid, std::string path, DirDetails &details);
 virtual int RequestDirDetails(void *ref, DirDetails &details, uint32_t flags);
 
-	/* Search Interface */
 virtual int SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results);
 virtual int SearchBoolExp(Expression * exp, std::list<FileDetail> &results);
 
-	/* Utility Functions */
+/***
+ * Utility Functions 
+ ***/
 virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath);
 virtual void ForceDirectoryCheck();
 virtual bool InDirectoryCheck();
 
-	/* Directory Handling */
+/***
+ * Directory Handling 
+ ***/
 virtual void	setDownloadDirectory(std::string path);
 virtual void	setPartialsDirectory(std::string path);
 virtual std::string getDownloadDirectory();
@@ -129,8 +151,6 @@ virtual bool	getSharedDirectories(std::list<std::string> &dirs);
 virtual bool 	addSharedDirectory(std::string dir);
 virtual bool 	removeSharedDirectory(std::string dir);
 
-virtual int 	reScanDirs();
-virtual int 	check_dBUpdate();
 
 	/***************************************************************/
 	/*************** Control Interface *****************************/
@@ -139,9 +159,20 @@ virtual int 	check_dBUpdate();
 	/***************************************************************/
 	/*************** Data Transfer Interface ***********************/
 	/***************************************************************/
+public:
+virtual bool    sendData(std::string peerId, std::string hash, uint64_t size,
+                        uint64_t offset, uint32_t chunksize, void *data);
+virtual bool    sendDataRequest(std::string peerId, 
+			std::string hash, uint64_t size,
+                        uint64_t offset, uint32_t chunksize);
 
-bool	requestData(std::string peerId, std::string hash, 
+	/*************** Internal Transfer Fns *************************/
-				uint64_t offset, uint32_t size);
+virtual int tick();
+
+private:
+bool	handleInputQueues();
+bool	handleCacheData();
+bool	handleFileData();
 
         /******************* p3 Config Overload ************************/
 	protected:
@@ -155,18 +186,34 @@ bool  loadConfigMap(std::map<std::string, std::string> &configMap);
         /******************* p3 Config Overload ************************/
 
 /*************************** p3 Config Overload ********************/
+
+	private:
+
+	/**** INTERNAL FUNCTIONS ***/
+virtual int 	reScanDirs();
+virtual int 	check_dBUpdate();
+
 	private:
 
 	/* no need for Mutex protection - 
 	 * as each component is protected independently.
 	 */
 
+        P3Interface *mP3iface;     /* XXX THIS NEEDS PROTECTION */
+        p3AuthMgr    *mAuthMgr;
+        p3ConnectMgr *mConnMgr;
+
 	CacheStrapper *mCacheStrapper;
+
 	ftController  *mFtController;
 	ftExtraList   *mFtExtra;
 
-        FileIndexStore *fiStore;
+	ftDataMultiplex *mFtDataplex;
-	FileIndexMonitor *fimon;
+
+        FileIndexStore *mFiStore;
+	FileIndexMonitor *mFiMon;
+
+	ftFileSearch   *mFtSearch;
 
 	RsMutex srvMutex;
 	std::string mConfigPath;

libretroshare/src/ft/fttransfermodule.cc

@@ -21,21 +21,29 @@
  *
  * Please report all bugs and problems to "retroshare@lunamutt.com".
  *
- */
+ */
-
+
-#include "fttransfermodule.h"
+#include "fttransfermodule.h"
-
+
-const uint32_t PQIPEER_OFFLINE_CHECK  = 120; /* check every 2 minutes */
+const uint32_t PQIPEER_OFFLINE_CHECK  = 120; /* check every 2 minutes */
-const uint32_t PQIPEER_DOWNLOAD_TIMEOUT  = 60; /* time it out, -> offline after 60 secs */
+const uint32_t PQIPEER_DOWNLOAD_TIMEOUT  = 60; /* time it out, -> offline after 60 secs */
-const uint32_t PQIPEER_DOWNLOAD_CHECK    = 10; /* desired delta = 10 secs */
+const uint32_t PQIPEER_DOWNLOAD_CHECK    = 10; /* desired delta = 10 secs */
-const uint32_t PQIPEER_DOWNLOAD_TOO_FAST = 8; /* 8 secs */
+const uint32_t PQIPEER_DOWNLOAD_TOO_FAST = 8; /* 8 secs */
-const uint32_t PQIPEER_DOWNLOAD_TOO_SLOW = 12; /* 12 secs */
+const uint32_t PQIPEER_DOWNLOAD_TOO_SLOW = 12; /* 12 secs */
-const uint32_t PQIPEER_DOWNLOAD_MIN_DELTA = 5; /* 5 secs */
+const uint32_t PQIPEER_DOWNLOAD_MIN_DELTA = 5; /* 5 secs */
-
+
-ftTransferModule::ftTransferModule(ftFileCreator *fc,ftClientModule *cm)
+ftTransferModule::ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm)
-	:mFileCreator(fc),mClientModule(cm),mFlag(0)
+	:mFileCreator(fc), mMultiplexor(dm), mFlag(0)
-{}
+{
-
+	mHash = mFileCreator->getHash();
+	mSize = mFileCreator->getFileSize();
+
+	// Dummy for Testing (should be handled independantly for 
+	// each peer.
+	mChunkSize = 10000;
+	return;
+}
+
 ftTransferModule::~ftTransferModule()
 {}
 
@@ -50,12 +58,12 @@ bool ftTransferModule::setFileSources(std::list<std::string> peerIds)
   return 1;
 }
 
-bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t maxRate);  //state = ONLINE/OFFLINE
+bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t maxRate)  //state = ONLINE/OFFLINE
 {
   bool found = false;
   std::list<std::string>::iterator it;
-  it = peerIds.begin();
+  it = mFileSources.begin();
-  while (( it != peerIds.end())&&(!found))
+  while (( it != mFileSources.end())&&(!found))
   {
     if ((*it) == peerId) 
       found = true;
@@ -66,12 +74,12 @@ bool ftTransferModule::setPeerState(std::string peerId,uint32_t state,uint32_t m
 
   std::map<std::string,peerInfo>::iterator mit;
   mit = mOnlinePeers.find(peerId);
-  if (mit == map::end)
+  if (mit == mOnlinePeers.end())
   {
     peerInfo pInfo;
     pInfo.state = state;
     pInfo.desiredRate = maxRate;
-    mOnlinePeers.insert(pair<std::string,peerInfo>(peerId,pInfo));
+    mOnlinePeers[peerId] = pInfo;
   }
   else
   {
@@ -86,26 +94,42 @@ uint32_t ftTransferModule::getDataRate(std::string peerId)
 {
   std::map<std::string,peerInfo>::iterator mit;
   mit = mOnlinePeers.find(peerId);
-  if (mit == map::end)
+  if (mit == mOnlinePeers.end())
     return 0;
   else
     return (mit->second).actualRate;
 }
 
-void ftTransferModule::requestData(uint64_t offset, uint32_t chunk_size)
+
-{
+  //interface to client module
-  mClientModule->requestData(mHash,offset,chunk_size);
+bool ftTransferModule::recvFileData(std::string peerId, uint64_t offset, 
-}
+			uint32_t chunk_size, void *data)
-
+{
+#ifdef FT_DEBUG
+	std::cerr << "ftTransferModule::recvFileData()";
+	std::cerr << " peerId: " << peerId;
+	std::cerr << " offset: " << offset;
+	std::cerr << " chunksize: " << chunk_size;
+	std::cerr << std::endl;
+#endif
+
+	return storeData(offset, chunk_size, data);
+}
+
+void ftTransferModule::requestData(std::string peerId, uint64_t offset, uint32_t chunk_size)
+{
+  mMultiplexor->sendDataRequest(peerId, mHash, mSize, offset,chunk_size);
+}
+
 bool ftTransferModule::getChunk(uint64_t &offset, uint32_t &chunk_size)
 {
   mFileCreator->getMissingChunk(offset, chunk_size);
 }
 
 bool ftTransferModule::storeData(uint64_t offset, uint32_t chunk_size,void *data)
-{
+{
-  mFileCreator->storeData(offset, chunk_size, data);
+	mFileCreator -> addFileData(offset, chunk_size, data);
-}
+}
 
 void ftTransferModule::queryInactive()
 {
@@ -133,7 +157,7 @@ void ftTransferModule::queryInactive()
       			(mit->second).req_size = mChunkSize;
       			(mit->second).lastTS = ts;
       			(mit->second).state = PQIPEER_DOWNLOADING;
-      			requestData(mOffset,mChunkSize);
+      			requestData(mit->first, mOffset,mChunkSize);
       		}
       	  else mFlag = 1;
       	}
@@ -142,7 +166,7 @@ void ftTransferModule::queryInactive()
       //file request has been sent to peer side, but no response received yet  
       case PQIPEER_DOWNLOADING:
       	if (ts - ((mit->second).lastTS) > PQIPEER_DOWNLOAD_CHECK)
-      		requestData((mit->second).req_loc,(mit->second).req_size);  //give a push
+      		requestData(mit->first, (mit->second).req_loc,(mit->second).req_size);  //give a push
         break;
       
       //file response has been received or peer side is just ready for download  
@@ -154,7 +178,7 @@ void ftTransferModule::queryInactive()
      			(mit->second).req_size = mChunkSize;
      			(mit->second).lastTS = ts;
      			(mit->second).state = PQIPEER_DOWNLOADING;
-     			requestData(mOffset,mChunkSize);
+     			requestData(mit->first, mOffset,mChunkSize);
      		}
      	  else mFlag = 1;        	
       	break;
@@ -192,7 +216,7 @@ bool ftTransferModule::resumeTransfer()
   return 1;
 }
 
-void ftTransferModule::completeFileTransfer()
+bool ftTransferModule::completeFileTransfer()
 {
 }	
 
@@ -205,3 +229,11 @@ int ftTransferModule::tick()
     
   return 0;
 }
+
+
+void ftTransferModule::adjustSpeed()
+{
+	return;
+}
+
+

libretroshare/src/ft/fttransfermodule.h

@@ -34,11 +34,16 @@
  * It must be able to cope with varying data rates and dropped peers without flooding the system with too many requests.
  *
  */
-
+
-#include <map>
+#include <map>
-#include <list>
+#include <list>
-#include <ftFileCreator.h>
+#include <string>
-
+
+#include "ft/ftfilecreator.h"
+#include "ft/ftdatamultiplex.h"
+
+#include "util/rsthreads.h"
+
 const int  PQIPEER_INIT                 = 0x0000;
 const int  PQIPEER_NOT_ONLINE           = 0x0001;
 const int  PQIPEER_DOWNLOADING          = 0x0002;
@@ -47,13 +52,15 @@ const int  PQIPEER_SUSPEND              = 0x0010;
 
 class Request
 {
+	public:
   uint64_t offset;
   uint32_t chunkSize;
 };
 
 class peerInfo
 {
-  std:string peerId;
+	public:
+  std::string peerId;
   uint32_t state;
   uint32_t desiredRate;
   Request lastRequest;
@@ -66,50 +73,55 @@ class peerInfo
   time_t lastTS;
 };
 
-class ftTransferModule 
+class ftTransferModule 
-{
+{
-public:
+public:
-  ftTransferModule(ftFileCreator *fc,ftClientModule *cm);
+  ftTransferModule(ftFileCreator *fc, ftDataMultiplex *dm);
-  ~ftTransferModule();
+  ~ftTransferModule();
-
+
-  //interface to download controller
+  //interface to download controller
-  bool setFileSources(std::list<std::string> peerIds);
+  bool setFileSources(std::list<std::string> peerIds);
-  bool setPeerState(std::string peerId,uint32_t state,uint32_t maxRate);  //state = ONLINE/OFFLINE
+  bool setPeerState(std::string peerId,uint32_t state,uint32_t maxRate);  //state = ONLINE/OFFLINE
-  uint32_t getDataRate(std::string peerId);
+  uint32_t getDataRate(std::string peerId);
-  bool stopTransfer();
+  bool stopTransfer();
-  bool resumeTransfer();
+  bool resumeTransfer();
-  bool completeFileTransfer();
+  bool completeFileTransfer();
-
+
-  //interface to client module
+  //interface to multiplex module
-  bool recvFileData(uint64_t offset, uint32_t chunk_size, void *data);  //called by ftClientModule when data arrives
+  bool recvFileData(std::string peerId, uint64_t offset, 
-  void requestData(uint64_t offset, uint32_t chunk_size);
+			uint32_t chunk_size, void *data);
-
+  void requestData(std::string peerId, uint64_t offset, uint32_t chunk_size);
-  //interface to file creator , just wrapper functions that call the ftFileCreator
+
-  bool getChunk(uint64_t &offset, uint32_t &chunk_size);
+  //interface to file creator
-  bool storeData(uint64_t offset, uint32_t chunk_size, void *data);
+  bool getChunk(uint64_t &offset, uint32_t &chunk_size);
-
+  bool storeData(uint64_t offset, uint32_t chunk_size, void *data);
-  //internal used functions
+
-  void queryInactive();
+  int tick();
-  
+
-  int tick();
+  std::string hash() { return mHash; }
-  
+  uint64_t    size() { return mSize; }
-  /* add by DrBob for interfaces */
+ 
-  std::string hash() { return mHash; }
+  //internal used functions
-  uint64_t    size() { return mSize; }
+  void queryInactive();
-  
+  void adjustSpeed();
-public:
+
-  ftFileCreator  *mFileCreator;
+private:
-  ftClientModule *mClientModule;
+
-
+  /* These have independent Mutexes / are const locally (no Mutex protection)*/
-private:
+  ftFileCreator *mFileCreator;
-  std::string mHash;
+  ftDataMultiplex *mMultiplexor;
-  uint64_t    mSize;
+
-  std::list<std::string>         mFileSources;
+  std::string mHash;
-  std::map<std::string,peerInfo> mOnlinePeers;
+  uint64_t    mSize;
-  	
+
-  uint64_t mOffset;
+  RsMutex tfMtx; /* below is mutex protected */
-  uint32_t mChunkSize;	
+
-  bool     mFlag;  //1:transfer complete, 0: not complete
+  std::list<std::string>         mFileSources;
-};
+  std::map<std::string,peerInfo> mOnlinePeers;
-
+  	
-#endif  //FT_TRANSFER_MODULE_HEADER
+  uint64_t mOffset;
+  uint32_t mChunkSize;
+  bool     mFlag;  //1:transfer complete, 0: not complete
+};
+
+#endif  //FT_TRANSFER_MODULE_HEADER

libretroshare/src/rsiface/rsfiles.h

@@ -51,19 +51,22 @@ const uint32_t RS_FILE_CTRL_STREAM_VIDEO = 0x0006;
 
 /************************************
  * Used To indicate where to search.
+ *
+ * The Order of these is very important, 
+ * it specifies the search order too.
+ *
  */
 
-const uint32_t RS_FILE_HINTS_MASK	 = 0x00ff;
+const uint32_t RS_FILE_HINTS_MASK	 = 0x00ffffff;
 
-const uint32_t RS_FILE_HINTS_CACHE	 = 0x0001;
+const uint32_t RS_FILE_HINTS_CACHE	 = 0x00000001;
-const uint32_t RS_FILE_HINTS_EXTRA	 = 0x0002;
+const uint32_t RS_FILE_HINTS_EXTRA	 = 0x00000002;
-const uint32_t RS_FILE_HINTS_LOCAL	 = 0x0004;
+const uint32_t RS_FILE_HINTS_LOCAL	 = 0x00000004;
-const uint32_t RS_FILE_HINTS_REMOTE	 = 0x0008;
+const uint32_t RS_FILE_HINTS_REMOTE	 = 0x00000008;
-const uint32_t RS_FILE_HINTS_DOWNLOAD	 = 0x0010;
+const uint32_t RS_FILE_HINTS_DOWNLOAD	 = 0x00000010;
-const uint32_t RS_FILE_HINTS_UPLOAD	 = 0x0020;
+const uint32_t RS_FILE_HINTS_UPLOAD	 = 0x00000020;
-
-const uint32_t RS_FILE_HINTS_SPEC_ONLY	 = 0x1000;
 
+const uint32_t RS_FILE_HINTS_SPEC_ONLY	 = 0x01000000;
 
 
 const uint32_t RS_FILE_EXTRA_DELETE	 = 0x0010;
@@ -79,43 +82,27 @@ virtual ~RsFiles() { return; }
 /****************************************/
 	/* download */
 
-/* Required Interfaces ......
- *
- * 1) Access to downloading / uploading files.
- */
-
-/* get Details of File Transfers */
-virtual bool FileDownloads(std::list<std::string> &hashs) = 0;
-virtual bool FileUploads(std::list<std::string> &hashs) = 0;
-virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0;
-
-
-/*
- * 2) Control of Downloads.
- *
- */
-
 
+/***
+ *  Control of Downloads.
+ ***/
 virtual bool FileRequest(std::string fname, std::string hash, 
 		uint32_t size, std::string dest, uint32_t flags) = 0;
 virtual bool FileCancel(std::string hash) = 0;
 virtual bool FileControl(std::string hash, uint32_t flags) = 0;
 virtual bool FileClearCompleted() = 0;
 
+/*** 
+ * Download / Upload Details.
+ ***/
+virtual bool FileDownloads(std::list<std::string> &hashs) = 0;
+virtual bool FileUploads(std::list<std::string> &hashs) = 0;
+virtual bool FileDetails(std::string hash, uint32_t hintflags, FileInfo &info) = 0;
 
-/*
- * 3) Addition of Extra Files... From File System
- * These are Hashed and stored in the 'Hidden Files' section
- * which can only be accessed if you know the hash.
- *
- * FileHash is called to start the hashing process, 
- * and add the file to the HiddenStore.
- *
- * FileHashStatus is called to lookup files 
- * and see if the hashing is completed.
- */
 
-/* Access ftExtraList - Details */
+/***
+ * Extra List Access
+ ***/
 virtual bool ExtraFileAdd(std::string fname, std::string hash, uint32_t size,
 				uint32_t period, uint32_t flags) = 0;
 virtual bool ExtraFileRemove(std::string hash, uint32_t flags) = 0;
@@ -124,25 +111,25 @@ virtual bool ExtraFileHash(std::string localpath,
 virtual bool ExtraFileStatus(std::string localpath, FileInfo &info) = 0;
 
 
-/*
+/***
- * 4) Search and Listing Interface 
+ * Directory Listing / Search Interface
  */
-
-/* Directory Listing / Search Interface */
 virtual int RequestDirDetails(std::string uid, std::string path, DirDetails &details) = 0;
 virtual int RequestDirDetails(void *ref, DirDetails &details, uint32_t flags) = 0;
 
 virtual int SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results) = 0;
 virtual int SearchBoolExp(Expression * exp, std::list<FileDetail> &results) = 0;
 
-/*
+/***
- * 5) Directory Control / Shared Files Utility Functions.
+ * Utility Functions.
- */
+ ***/
-
 virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath) = 0;
 virtual void ForceDirectoryCheck() = 0;
 virtual bool InDirectoryCheck() = 0;
 
+/***
+ * Directory Control
+ ***/
 virtual void    setDownloadDirectory(std::string path) = 0;
 virtual void    setPartialsDirectory(std::string path) = 0;
 virtual std::string getDownloadDirectory() = 0;

libretroshare/src/serialiser/rstlvtypes.cc

@@ -33,6 +33,10 @@
 #include <iomanip>
 #include <iostream>
 
+void  	RsTlvItem::TlvShallowClear()
+{
+	TlvClear(); /* unless overloaded! */
+}
 
 std::ostream &RsTlvItem::printBase(std::ostream &out, std::string clsName, uint16_t indent)
 {
@@ -102,6 +106,11 @@ void RsTlvBinaryData::TlvClear()
 	{
 		free(bin_data);
 	}
+	TlvShallowClear();
+}
+
+void  	RsTlvBinaryData::TlvShallowClear()
+{
 	bin_data = NULL;
 	bin_len = 0;
 }

libretroshare/src/serialiser/rstlvtypes.h

@@ -47,6 +47,7 @@ class RsTlvItem
 virtual ~RsTlvItem() { return; }
 virtual uint16_t TlvSize() = 0;
 virtual void	 TlvClear() = 0;
+virtual	void	 TlvShallowClear(); /*! Don't delete allocated data */
 virtual bool     SetTlv(void *data, uint32_t size, uint32_t *offset) = 0; /* serialise   */
 virtual bool     GetTlv(void *data, uint32_t size, uint32_t *offset) = 0; /* deserialise */
 virtual std::ostream &print(std::ostream &out, uint16_t indent) = 0;
@@ -66,6 +67,8 @@ class RsTlvBinaryData: public RsTlvItem
 virtual ~RsTlvBinaryData();
 virtual	uint16_t TlvSize();
 virtual	void	 TlvClear(); /*! Initialize fields to empty legal values ( "0", "", etc) */
+virtual	void	 TlvShallowClear(); /*! Don't delete the binary data */
+
  /// Serialise.
 /*! Serialise Tlv to buffer(*data) of 'size' bytes starting at *offset */
 virtual bool     SetTlv(void *data, uint32_t size, uint32_t *offset);

libretroshare/src/util/rsthreads.h

@@ -76,13 +76,17 @@ virtual void run() = 0; /* called once the thread is started */
 };
 
 
-class RsQueueThread
+class RsQueueThread: public RsThread
 {
+	public:
+
 	RsQueueThread(uint32_t min, uint32_t max, double relaxFactor );
 virtual ~RsQueueThread() { return; }
 
 virtual void run();
 
+	protected:
+
 virtual bool workQueued() = 0;
 virtual bool doWork() = 0;