Created V0.3.x branch and moved the head into the trunk directory.

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@246 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/dbase/Makefile

@@ -0,0 +1,53 @@
+
+RS_TOP_DIR = ..
+include ../make.opt
+
+OBJ = filedex.o filelook.o \
+	findex.o fimonitor.o cachestrapper.o fistore.o \
+	rsexpr.o
+
+EXEC = ftest looktest fitest2 fisavetest ficachetest searchtest
+
+ifeq ($(OS),Linux)
+	EXEC += fimontest 
+endif
+
+all : $(OBJ) librs $(EXEC)
+
+librs:  $(OBJ)
+	$(AR) r $(LIBRS) $(OBJ)
+	$(RANLIB) $(LIBRS)
+
+ftest: $(OBJ) ftest.o
+	$(CC) $(CFLAGS) -o ftest $(OBJ) ftest.o $(RSLIBS)
+
+looktest: $(OBJ) looktest.o
+	$(CC) $(CFLAGS) -o looktest $(OBJ) looktest.o $(RSLIBS)
+
+fitest2 : fitest2.o $(OBJ)
+	$(CC) $(CFLAGS) -o fitest2 fitest2.o $(OBJ) $(RSLIBS)
+
+fisavetest : fisavetest.o $(OBJ)
+	$(CC) $(CFLAGS) -o fisavetest fisavetest.o $(OBJ) $(RSLIBS)
+
+fimontest : fimontest.o $(OBJ)
+	$(CC) $(CFLAGS) -o fimontest fimontest.o $(OBJ) $(RSLIBS)
+
+ficachetest : ficachetest.o $(OBJ)
+	$(CC) $(CFLAGS) -o ficachetest ficachetest.o $(OBJ) $(RSLIBS)
+
+searchtest : searchtest.o $(OBJ)
+	$(CC) $(CFLAGS) -o searchtest searchtest.o $(OBJ) $(RSLIBS)
+
+
+.cc.o:
+	$(CC) $(CFLAGS) -c $<
+
+clean:
+	-/bin/rm $(OBJ) ftest.o looktest.o
+	-/bin/rm fitest2.o fisavetest.o fimontest.o
+	-/bin/rm ficachetest.o searchtest.o
+
+clobber: clean
+	-/bin/rm $(EXEC)
+

libretroshare/src/dbase/cachestrapper.cc

@@ -0,0 +1,721 @@
+/*
+ * RetroShare FileCache Module: cachestrapper.cc
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "dbase/cachestrapper.h"
+
+#include <iostream>
+#include <sstream>
+#include <iomanip>
+
+/**
+ * #define CS_DEBUG 1
+ */
+
+bool operator<(const CacheId &a, const CacheId &b)
+{
+        if (a.type == b.type)
+		return (a.subid < b.subid);
+	return (a.type < b.type);
+}
+
+bool operator<(const CachePair &a, const CachePair &b)
+{
+	return (a.id < b.id);
+}
+
+
+std::ostream &operator<<(std::ostream &out, const CacheData &d)
+{
+	out << "[ p: " << d.pid << " id: <" << d.cid.type << "," << d.cid.subid;
+	out << "> #" << d.hash << " size: " << d.size;
+	out << " \"" << d.name << "\"@\"" << d.path;
+	out << "\" ]";
+	return out;
+}
+
+/********************************* Cache Store / Source **************************
+ * This is a generic interface which interacts with the FileTransfer Unit
+ * to collect Data to be Cached.
+ *
+ ********************************* Cache Store / Source *************************/
+
+CacheSource::CacheSource(uint16_t t, bool m, std::string cachedir)
+	:cacheType(t), multiCache(m), cacheDir(cachedir)
+	{ 
+		return; 
+	}
+
+	/* Mutex Stuff -> to be done... */
+void    CacheSource::lockData()
+{
+#ifdef CS_DEBUG
+	std::cerr << "CacheSource::lockData()" << std::endl;
+#endif
+	cMutex.lock();
+}
+
+void    CacheSource::unlockData()
+{
+#ifdef CS_DEBUG
+	std::cerr << "CacheSource::unlockData()" << std::endl;
+#endif
+	cMutex.unlock();
+}
+	
+	/* to be overloaded for inherited Classes */
+bool    CacheSource::loadCache(const CacheData &data)
+{
+	return refreshCache(data);
+}
+	
+        /* control Caches available */
+bool    CacheSource::refreshCache(const CacheData &data)
+{
+	lockData(); /* LOCK MUTEX */
+
+	bool ret = false;
+	if (data.cid.type == getCacheType())
+	{
+		if (isMultiCache())
+		{
+			caches[data.cid.subid] = data;
+		}
+		else
+		{
+			caches[0] = data;
+		}
+		ret = true;
+	}
+
+	unlockData(); /* UNLOCK MUTEX */
+	return ret;
+}
+		
+bool    CacheSource::clearCache(CacheId id)
+{
+	lockData(); /* LOCK MUTEX */
+
+	bool ret = false;
+	if (id.type == getCacheType())
+	{
+		CacheSet::iterator it;
+		if (caches.end() != (it = caches.find(id.subid)))
+		{
+			caches.erase(it);
+			ret = true;
+		}
+	}
+
+	unlockData(); /* UNLOCK MUTEX */
+	return ret;
+}
+		
+bool    CacheSource::cachesAvailable(RsPeerId pid, std::map<CacheId, CacheData> &ids)
+{
+	lockData(); /* LOCK MUTEX */
+
+	/* can overwrite for more control! */
+	CacheSet::iterator it;
+	for(it = caches.begin(); it != caches.end(); it++)
+	{
+		ids[(it->second).cid] = it->second;
+	}
+	bool ret = (caches.size() > 0);
+
+	unlockData(); /* UNLOCK MUTEX */
+	return ret;
+
+}
+
+
+bool    CacheSource::findCache(std::string hash, CacheData &data)
+{
+	lockData(); /* LOCK MUTEX */
+
+	bool found = false;
+	CacheSet::iterator it;
+	for(it = caches.begin(); it != caches.end(); it++)
+	{
+		if (hash == (it->second).hash)
+		{
+			data = it->second;
+			found = true;
+			break;
+		}
+	}
+
+	unlockData(); /* UNLOCK MUTEX */
+
+	return found;
+}
+
+		
+void    CacheSource::listCaches(std::ostream &out)
+{
+	lockData(); /* LOCK MUTEX */
+
+	/* can overwrite for more control! */
+	CacheSet::iterator it;
+	out << "CacheSource::listCaches() [" << getCacheType();
+	out << "] Total: " << caches.size() << std::endl;
+	int i;
+	for(i = 0, it = caches.begin(); it != caches.end(); it++, i++)
+	{
+		out << "\tC[" << i << "] : " << it->second << std::endl;
+	}
+
+	unlockData(); /* UNLOCK MUTEX */
+	return;
+}
+
+
+CacheStore::CacheStore(uint16_t t, bool m, CacheTransfer *cft, std::string cachedir)
+	:cacheType(t), multiCache(m), cacheTransfer(cft), cacheDir(cachedir)
+	{ 
+		/* not much */
+		return; 
+	}
+
+	/* Mutex Stuff -> to be done... */
+void    CacheStore::lockData()
+{
+#ifdef CS_DEBUG
+	std::cerr << "CacheStore::lockData()" << std::endl;
+#endif
+	cMutex.lock();
+}
+
+void    CacheStore::unlockData()
+{
+#ifdef CS_DEBUG
+	std::cerr << "CacheStore::unlockData()" << std::endl;
+#endif
+	cMutex.unlock();
+}
+		
+void    CacheStore::listCaches(std::ostream &out)
+{
+	lockData(); /* LOCK MUTEX */
+
+	/* can overwrite for more control! */
+	std::map<RsPeerId, CacheSet>::iterator pit;
+	out << "CacheStore::listCaches() [" << getCacheType();
+	out << "] Total People: " << caches.size();
+	out << std::endl;
+	for(pit = caches.begin(); pit != caches.end(); pit++)
+	{
+		CacheSet::iterator it;
+		out << "\tTotal for [" << pit->first << "] : " << (pit->second).size();
+		out << std::endl;
+		for(it = (pit->second).begin(); it != (pit->second).end(); it++)
+		{
+			out << "\t\t" << it->second;
+			out << std::endl;
+		}
+	}
+
+	unlockData(); /* UNLOCK MUTEX */
+	return;
+}
+
+
+	/* look for stored data. using pic/cid in CacheData
+	 */
+bool	CacheStore::getStoredCache(CacheData &data)
+{
+	lockData(); /* LOCK MUTEX */
+
+	bool ok = locked_getStoredCache(data);
+
+	unlockData(); /* UNLOCK MUTEX */
+	return ok;
+}
+
+
+bool	CacheStore::locked_getStoredCache(CacheData &data)
+{
+	if (data.cid.type != getCacheType())
+	{
+		return false;
+	}
+
+	std::map<RsPeerId, CacheSet>::iterator pit;
+	if (caches.end() == (pit = caches.find(data.pid)))
+	{
+		return false;
+	}
+
+	CacheSet::iterator cit;
+	if (isMultiCache())
+	{
+		/* look for subid */
+		if ((pit->second).end() == 
+			(cit = (pit->second).find(data.cid.subid)))
+		{
+			return false;
+		}
+	}
+	else
+	{
+		if ((pit->second).end() == 
+			(cit = (pit->second).find(0)))
+		{
+			return false;
+		}
+	}
+
+	/* we found it! (cit) */
+	data = cit->second;
+
+	return true;
+}
+
+
+
+	/* input from CacheStrapper.
+	 * check if we want to download it...
+	 * determine the new name/path
+	 * then request it.
+	 */
+void	CacheStore::availableCache(const CacheData &data)
+{
+	std::cerr << "CacheStore::availableCache() :" << data << std::endl;
+	/* basic checks */
+	lockData(); /* LOCK MUTEX */
+
+	bool rightCache = (data.cid.type == getCacheType());
+
+	unlockData(); /* UNLOCK MUTEX */
+
+	if (!rightCache)
+	{
+		return; /* bad id */
+	}
+
+	/* These Functions lock the Mutex themselves
+	 */
+
+	if (!fetchCache(data))
+	{
+		return; /* ignore it */
+	}
+
+	CacheData rData = data;
+
+	/* get new name */
+	if (!nameCache(rData))
+	{
+		return; /* error naming */
+	}
+
+	/* request it */
+	cacheTransfer -> RequestCache(rData, this);
+
+	/* will get callback when it is complete */
+	return;
+}
+
+
+	/* called when the download is completed ... updates internal data */
+void 	CacheStore::downloadedCache(const CacheData &data)
+{
+	std::cerr << "CacheStore::downloadedCache() :" << data << std::endl;
+	/* updates data */
+	if (!loadCache(data))
+	{
+		return;
+	}
+}
+	/* called when the download is completed ... updates internal data */
+void 	CacheStore::failedCache(const CacheData &data)
+{
+	std::cerr << "CacheStore::failedCache() :" << data << std::endl;
+	return;
+}
+
+
+	/* virtual function overloaded by cache implementor */
+bool    CacheStore::fetchCache(const CacheData &data)
+{
+	/* should we fetch it? */
+	std::cerr << "CacheStore::fetchCache() tofetch?:" << data << std::endl;
+
+	CacheData incache = data;
+
+	lockData(); /* LOCK MUTEX */
+
+	bool haveCache = ((locked_getStoredCache(incache)) && (data.hash == incache.hash));
+
+	unlockData(); /* UNLOCK MUTEX */
+
+
+	if (haveCache)
+	{
+		std::cerr << "CacheStore::fetchCache() Already have it: false" << std::endl;
+		return false;
+	}
+
+	std::cerr << "CacheStore::fetchCache() Missing this cache: true" << std::endl;
+	return true;
+}
+
+
+int     CacheStore::nameCache(CacheData &data)
+{
+	/* name it... */
+	lockData(); /* LOCK MUTEX */
+
+
+	std::cerr << "CacheStore::nameCache() for:" << data << std::endl;
+	data.name = data.hash;
+	data.path = getCacheDir();
+	std::cerr << "CacheStore::nameCache() done:" << data << std::endl;
+
+	unlockData(); /* UNLOCK MUTEX */
+
+	return 1;
+}
+
+
+int     CacheStore::loadCache(const CacheData &data)
+{
+	/* attempt to load -> dummy function */
+	std::cerr << "CacheStore::loadCache() Dummy Load for:" << data << std::endl;
+
+	lockData(); /* LOCK MUTEX */
+
+	locked_storeCacheEntry(data);
+
+	unlockData(); /* UNLOCK MUTEX */
+	
+	return 1;
+}
+
+/* This function is called to store Cache Entry in the CacheStore Table.
+ * it must be called from within a Mutex Lock....
+ *
+ * It doesn't lock itself -> to avoid race conditions 
+ */
+
+void 	CacheStore::locked_storeCacheEntry(const CacheData &data)
+{
+	/* store what we loaded - overwriting if necessary */
+	std::map<RsPeerId, CacheSet>::iterator pit;
+	if (caches.end() == (pit = caches.find(data.pid)))
+	{
+		/* add in a new CacheSet */
+		CacheSet emptySet;
+		caches[data.pid] = emptySet;
+
+		pit = caches.find(data.pid);
+	}
+
+	if (isMultiCache())
+	{
+		(pit->second)[data.cid.subid] = data;
+	}
+	else
+	{
+		(pit->second)[0] = data;
+	}
+	return;
+}
+
+
+/********************************* CacheStrapper *********************************
+ * This is the bit which handles queries
+ *
+ ********************************* CacheStrapper ********************************/
+
+CacheStrapper::CacheStrapper(RsPeerId id, time_t period)
+	:ownId(id), queryPeriod(period)
+{
+	return;
+}
+
+
+void	CacheStrapper::addCachePair(CachePair set)
+{
+	caches[set.id.type] = set;
+}
+
+
+        /* from pqimonclient */
+void    CacheStrapper::monUpdate(const std::list<pqipeer> &plist)
+{
+	std::list<pqipeer>::const_iterator it;
+	std::map<RsPeerId, CacheTS>::iterator mit;
+	for(it = plist.begin(); it != plist.end(); it++)
+	{
+		if (status.end() == (mit = status.find(it->id)))
+		{
+			addPeerId(it->id);
+		}
+	}
+}
+
+
+void	CacheStrapper::addPeerId(RsPeerId pid)
+{
+	std::map<RsPeerId, CacheTS>::iterator it;
+
+	/* just reset it for the moment */
+	CacheTS ts;
+	ts.query = 0;
+	ts.answer = 0;
+
+	status[pid] = ts;
+}
+
+bool	CacheStrapper::removePeerId(RsPeerId pid)
+{
+	std::map<RsPeerId, CacheTS>::iterator it;
+	if (status.end() != (it = status.find(pid)))
+	{
+		status.erase(it);
+		return true;
+	}
+	return false; 
+}
+
+
+	/* pass to correct CacheSet */
+void	CacheStrapper::recvCacheResponse(CacheData &data, time_t ts)
+{
+	/* update internal data first */
+	std::map<RsPeerId, CacheTS>::iterator it;
+	if (status.end() == status.find(data.pid))
+	{
+		/* add it in */
+		CacheTS d;
+		d.query = 0;
+		d.answer = 0;
+
+		status[data.pid] = d;
+	}
+
+	it = status.find(data.pid); /* will always succeed */
+
+	/* update status */
+	(it -> second).answer = ts;
+
+	/* find cache store */
+	std::map<uint16_t, CachePair>::iterator it2;
+	if (caches.end() == (it2 = caches.find(data.cid.type)))
+	{
+		/* error - don't have this type of cache */
+		return;
+	}
+
+	/* notify the CacheStore */
+	(it2 -> second).store -> availableCache(data);
+}
+
+
+	/* generate periodically or at a change */
+bool    CacheStrapper::sendCacheQuery(std::list<RsPeerId> &id, time_t ts)
+{
+	/* iterate through peers, and see who we haven't got an answer from recently */
+	std::map<RsPeerId, CacheTS>::iterator it;
+	for(it = status.begin(); it != status.end(); it++)
+	{
+		if ((ts - (it->second).query) > queryPeriod)
+		{
+			/* query this one */
+			id.push_back(it->first);
+			(it->second).query = ts;
+		}
+	}
+	return (id.size() > 0);
+}
+
+
+void    CacheStrapper::handleCacheQuery(RsPeerId id, std::map<CacheId,CacheData> &hashs)
+{
+	/* basic version just iterates through ....
+	 * more complex could decide who gets what!
+	 *
+	 * or that can be handled on a cache by cache basis.
+	 */
+
+	std::map<uint16_t, CachePair>::iterator it;
+	for(it = caches.begin(); it != caches.end(); it++)
+	{
+		(it->second).source -> cachesAvailable(id, hashs);
+	}
+	return;
+}
+
+void    CacheStrapper::listCaches(std::ostream &out)
+{
+	/* can overwrite for more control! */
+	std::map<uint16_t, CachePair>::iterator it;
+	out << "CacheStrapper::listCaches() [" << ownId;
+	out << "] Total Peers: " << status.size() << " Total Caches: " << caches.size();
+	out << std::endl;
+	for(it = caches.begin(); it != caches.end(); it++)
+	{
+		out << "CacheType: " << it->first;
+		out << std::endl;
+
+		(it->second).source->listCaches(out);
+		(it->second).store->listCaches(out);
+		out << std::endl;
+	}
+	return;
+}
+
+void    CacheStrapper::listPeerStatus(std::ostream &out)
+{
+	std::map<RsPeerId, CacheTS>::iterator it;
+	out << "CacheStrapper::listPeerStatus() [" << ownId;
+	out << "] Total Peers: " << status.size() << " Total Caches: " << caches.size();
+	out << std::endl;
+	for(it = status.begin(); it != status.end(); it++)
+	{
+		out << "Peer: " << it->first;
+		out << " Query: " << (it->second).query;
+		out << " Answer: " << (it->second).answer;
+		out << std::endl;
+	}
+	return;
+}
+
+
+bool    CacheStrapper::findCache(std::string hash, CacheData &data)
+{
+	/* can overwrite for more control! */
+	std::map<uint16_t, CachePair>::iterator it;
+	for(it = caches.begin(); it != caches.end(); it++)
+	{
+		if ((it->second).source->findCache(hash, data))
+		{
+			return true;
+		}
+	}
+	return false;
+}
+
+
+/********************************* CacheStrapper *********************************
+ * This is the bit which handles queries
+ *
+ ********************************* CacheStrapper ********************************/
+
+
+/* request from CacheStore */
+bool CacheTransfer::RequestCache(CacheData &data, CacheStore *cbStore)
+{
+	/* store request */
+	cbData[data.hash] = data;
+	cbStores[data.hash] = cbStore;
+
+	/* request data */
+	RequestCacheFile(data.pid, data.path, data.hash, data.size);
+
+	/* wait for answer */
+	return true;
+}
+
+
+/* to be overloaded */
+bool CacheTransfer::RequestCacheFile(RsPeerId id, std::string path, std::string hash, uint32_t size)
+{
+	std::cerr << "CacheTransfer::RequestCacheFile() : from:" << id << " #";
+	std::cerr << hash << " size: " << size;
+	std::cerr << " savepath: " << path << std::endl;
+	std::cerr << "CacheTransfer::RequestCacheFile() Dummy... saying completed";
+	std::cerr << std::endl;
+
+	/* just tell them we've completed! */
+	CompletedCache(hash);
+	return true;
+}
+
+
+/* internal completion -> does cb */
+bool CacheTransfer::CompletedCache(std::string hash)
+{
+	std::map<std::string, CacheData>::iterator dit;
+	std::map<std::string, CacheStore *>::iterator sit;
+
+	/* find in store.... */
+	sit = cbStores.find(hash);
+	dit = cbData.find(hash);
+
+	if ((sit == cbStores.end()) || (dit == cbData.end()))
+	{
+		return false;
+	}
+	
+	/* callback */
+	(sit -> second) -> downloadedCache(dit->second);
+
+	/* clean up store */
+	cbStores.erase(sit);
+	cbData.erase(dit);
+
+	return true;
+}
+
+/* internal completion -> does cb */
+bool CacheTransfer::FailedCache(std::string hash)
+{
+	std::map<std::string, CacheData>::iterator dit;
+	std::map<std::string, CacheStore *>::iterator sit;
+
+	/* find in store.... */
+	sit = cbStores.find(hash);
+	dit = cbData.find(hash);
+
+	if ((sit == cbStores.end()) || (dit == cbData.end()))
+	{
+		return false;
+	}
+	
+	/* callback */
+	(sit -> second) -> failedCache(dit->second);
+
+	/* clean up store */
+	cbStores.erase(sit);
+	cbData.erase(dit);
+
+	return true;
+}
+
+
+bool    CacheTransfer::FindCacheFile(std::string hash, std::string &path, uint32_t &size)
+{
+	CacheData data;
+	if (strapper->findCache(hash, data))
+	{
+		path = data.path + "/" + data.name;
+		size = data.size;
+		return true;
+	}
+
+	return false;
+}
+
+
+

libretroshare/src/dbase/cachestrapper.h

@@ -0,0 +1,321 @@
+/*
+ * RetroShare FileCache Module: cachestrapper.h
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef MRK_CACHE_STRAPPER_H
+#define MRK_CACHE_STRAPPER_H
+
+#include <list>
+#include <map>
+#include <string>
+#include <iostream>
+#include "util/rsthreads.h"
+
+/******************* CacheStrapper and Related Classes *******************
+ * A generic Cache Update system.
+ *
+ * CacheStrapper: maintains a set of CacheSources, and CacheStores, 
+ *     queries and updates as new information arrives.
+ *
+ * CacheTransfer: Interface for FileTransfer Class to support.
+ *
+ * CacheSource: Base Class for cache data provider. eg. FileIndexMonitor.
+ * CacheStore: Base Class for data cache. eg. FileCache/Store.
+ *
+ * Still TODO:
+ * (1) Design and Implement the Upload side of CacheTransfer/CacheStrapper.
+ * (2) CacheStrapper:: Save / Load Cache lists....
+ * (3) Clean up lists, maps on shutdown etc.
+ * (4) Consider Mutexes for multithreaded operations.
+ * (5) Test the MultiSource/Store capabilities.
+ *
+ ******************* CacheStrapper and Related Classes *******************/
+
+
+class CacheTransfer; /* Interface for File Transfer */
+class CacheSource;   /* Interface for local File Index/Monitor */
+class CacheStore;    /* Interface for the actual Cache */
+class CacheStrapper; /* Controlling Class */
+
+/****
+typedef uint32_t RsPeerId;
+*****/
+typedef std::string RsPeerId;
+
+/******************************** CacheId ********************************/
+class CacheId
+{
+	public:
+	CacheId() :type(0), subid(0) { return; }
+	CacheId(uint16_t a, uint16_t b) :type(a), subid(b) { return; }
+	uint16_t type;
+	uint16_t subid;
+};
+
+
+bool operator<(const CacheId &a, const CacheId &b);
+
+#define CACHE_TYPE_FILE_INDEX	1
+#define CACHE_TYPE_WEB		2
+#define CACHE_TYPE_OTHER	4
+
+/* think of any others? */
+
+class CacheData
+{
+	public:
+
+	RsPeerId pid;
+	std::string pname; /* peer name (can be used by cachestore) */
+	CacheId  cid;
+	std::string path; 
+	std::string name;
+	std::string hash;
+	uint32_t size;
+	time_t recvd;
+};
+
+
+std::ostream &operator<<(std::ostream &out, const CacheData &d);
+
+/***************************** CacheTransfer *****************************/
+
+class CacheTransfer
+{
+	public:
+	CacheTransfer(CacheStrapper *cs) :strapper(cs) { return; }
+virtual ~CacheTransfer() {}
+
+	/* upload side of things .... searches through CacheStrapper. */
+bool    FindCacheFile(std::string hash, std::string &path, uint32_t &size);
+
+
+	/* At the download side RequestCache() => overloaded RequestCacheFile()
+	 * the class should then call CompletedCache() or FailedCache()
+	 */
+
+bool RequestCache(CacheData &data, CacheStore *cbStore); /* request from CacheStore */
+
+	protected:
+	/* to be overloaded */
+virtual bool RequestCacheFile(RsPeerId id, std::string path, std::string hash, uint32_t size); 
+
+bool CompletedCache(std::string hash);                   /* internal completion -> does cb */
+bool FailedCache(std::string hash);                      /* internal completion -> does cb */
+
+	private:
+
+	CacheStrapper *strapper;
+
+	std::map<std::string, CacheData>    cbData;
+	std::map<std::string, CacheStore *> cbStores;
+};
+
+
+
+/************************ CacheSource/CacheStore *************************/
+
+typedef std::map<uint16_t, CacheData> CacheSet;
+
+class CacheSource
+{
+	public:
+	CacheSource(uint16_t t, bool m, std::string cachedir);
+virtual ~CacheSource() {}
+
+	/* called to determine available cache for peer - 
+	 * default acceptable (returns all) 
+	 */
+virtual bool 	cachesAvailable(RsPeerId pid, std::map<CacheId, CacheData> &ids);
+
+	/* function called at startup to load from 
+	 * configuration file....
+	 * to be overloaded by inherited class 
+	 */
+virtual bool    loadCache(const CacheData &data);
+
+	/* control Caches available */
+bool    refreshCache(const CacheData &data);
+bool 	clearCache(CacheId id);
+
+	/* get private data */
+std::string getCacheDir()    { return cacheDir;   }
+bool        isMultiCache()   { return multiCache; }
+uint16_t    getCacheType()   { return cacheType;  }
+
+	/* display */
+void 	listCaches(std::ostream &out);
+
+	/* search */
+bool    findCache(std::string hash, CacheData &data);
+
+	protected:
+
+	/*** MUTEX LOCKING - TODO 
+	*/
+void	lockData();
+void	unlockData();
+
+	CacheSet caches;
+
+	private:
+
+	uint16_t cacheType;    /* for checking */
+	bool   multiCache;   /* do we care about subid's */
+
+	std::string cacheDir;
+	RsMutex cMutex;
+};
+
+
+class CacheStore
+{
+	public:
+
+	CacheStore(uint16_t t, bool m, CacheTransfer *cft, std::string cachedir);
+virtual ~CacheStore() {} 
+
+	/* current stored data */
+bool 	getStoredCache(CacheData &data); /* use pid/cid in data */
+
+	/* input from CacheStrapper -> store can then download new data */
+void	availableCache(const CacheData &data);
+
+	/* called when the download is completed ... updates internal data */
+void 	downloadedCache(const CacheData &data);
+
+	/* called if the download fails */
+void 	failedCache(const CacheData &data);
+
+	/* virtual functions overloaded by cache implementor */
+virtual bool fetchCache(const CacheData &data);   /* a question? */
+virtual int nameCache(CacheData &data);           /* fill in the name/path */
+virtual int loadCache(const CacheData &data);	  /* actual load, once data available */
+
+	/* get private data */
+std::string getCacheDir()    { return cacheDir;   }
+bool        isMultiCache()   { return multiCache; }
+uint16_t    getCacheType()   { return cacheType;  }
+
+	/* display */
+void 	listCaches(std::ostream &out);
+
+	protected:
+	/*** MUTEX LOCKING */
+void	lockData();
+void	unlockData();
+
+	/* This function is called to store Cache Entry in the CacheStore Table.
+	 * it must be called from within a Mutex Lock....
+	 *
+	 * It doesn't lock itself -> to avoid race conditions
+	 */
+void    locked_storeCacheEntry(const CacheData &data);
+bool    locked_getStoredCache(CacheData &data);
+
+	private:
+
+	uint16_t cacheType;    /* for checking */
+	bool     multiCache;   /* do we care about subid's */
+
+	CacheTransfer *cacheTransfer;
+	std::string cacheDir;
+
+	std::map<RsPeerId, CacheSet> caches;
+
+	RsMutex cMutex;
+};
+
+
+
+/***************************** CacheStrapper *****************************/
+
+
+/* Make Sure you get the Ids right! */
+class CachePair
+{
+	public:
+	CachePair()
+	:source(NULL), store(NULL), id(0, 0) { return; }
+
+	CachePair(CacheSource *a, CacheStore *b, CacheId c)
+	:source(a), store(b), id(c) { return; }
+
+	CacheSource *source;
+	CacheStore  *store;
+	CacheId     id;
+};
+
+
+bool operator<(const CachePair &a, const CachePair &b);
+
+class CacheTS
+{
+	public:
+
+	time_t query;
+	time_t answer;
+};
+
+#include "pqi/pqimon.h"
+
+class CacheStrapper: public pqimonclient
+{
+	public:
+	CacheStrapper(RsPeerId id, time_t period);
+virtual ~CacheStrapper() { return; }
+
+	/* from pqimonclient */
+virtual void monUpdate(const std::list<pqipeer> &plist);
+
+void	addCachePair(CachePair pair);
+
+void   addPeerId(RsPeerId pid);
+bool   removePeerId(RsPeerId pid);
+
+	/*** I/O (1) ***/
+				/* pass to correct CacheSet */
+void	recvCacheResponse(CacheData &date, time_t ts);  
+				/* generate periodically or at a change */
+bool    sendCacheQuery(std::list<RsPeerId> &id, time_t ts);		
+
+	/*** I/O (2) ***/
+				/* handle a DirQuery */
+void    handleCacheQuery(RsPeerId id, std::map<CacheId, CacheData> &data); 
+
+	/* search through CacheSources. */
+bool    findCache(std::string hash, CacheData &data);
+
+	/* display */
+void 	listCaches(std::ostream &out);
+void 	listPeerStatus(std::ostream &out);
+	
+	private:
+
+	std::map<RsPeerId, CacheTS> status;
+	std::map<uint16_t, CachePair> caches;
+	RsPeerId ownId;
+	time_t queryPeriod;
+};
+
+
+#endif

libretroshare/src/dbase/cachetest.h

@@ -0,0 +1,63 @@
+/*
+ * RetroShare FileCache Module: cachetest.h
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef MRK_TEST_CACHE_H
+#define MRK_TEST_CACHE_H
+
+#include "cachestrapper.h"
+
+#define TESTID 0xffff
+#define TESTID2 0xffee
+
+class CacheTestSource: public CacheSource
+{
+	public:
+	CacheTestSource(std::string dir)
+	:CacheSource(TESTID, false, dir) { return; }
+};
+
+class CacheTestStore: public CacheStore
+{
+	public:
+	CacheTestStore(CacheTransfer *cft, std::string dir)
+	:CacheStore(TESTID, false, cft, dir) { return; }
+};
+
+
+class CacheTestMultiSource: public CacheSource
+{
+	public:
+	CacheTestMultiSource(std::string dir)
+	:CacheSource(TESTID2, true, dir) { return; }
+};
+
+class CacheTestMultiStore: public CacheStore
+{
+	public:
+	CacheTestMultiStore(CacheTransfer *cft, std::string dir)
+	:CacheStore(TESTID2, true, cft, dir) { return; }
+};
+
+
+#endif
+

libretroshare/src/dbase/ficachetest.cc

@@ -0,0 +1,209 @@
+/*
+ * RetroShare FileCache Module: ficachetest.cc
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "cachestrapper.h"
+#include "cachetest.h"
+
+#include <iostream>
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+#ifndef WINDOWS_SYS
+#else
+	#include <windows.h>
+#endif
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+
+
+void handleQuery(CacheStrapper *csp, RsPeerId pid, 
+		std::map<RsPeerId, CacheStrapper *> &strappers);
+
+/* A simple test of the CacheStrapper Code.
+ *
+ * create 3 different CacheStrappers, each with a Source/Store Pair and Transfer Class.
+ * pass queries and responses between the CacheStrappers, 
+ * and ensure that the hashes in the Caches are updated.
+ *
+ */
+
+int main(int argc, char **argv)
+{
+
+	time_t period = 11;
+	RsPeerId  pid1("0x0101");
+	RsPeerId  pid2("0x0102");
+	RsPeerId  pid3("0x0103");
+
+	CacheStrapper sc1(pid1, period);
+	CacheStrapper sc2(pid2, period);
+	CacheStrapper sc3(pid3, period);
+	CacheTransfer ctt1(&sc1);
+	CacheTransfer ctt2(&sc2);
+	CacheTransfer ctt3(&sc3);
+
+	std::map<RsPeerId, CacheStrapper *> strappers;
+	strappers[pid1] = &sc1;
+	strappers[pid2] = &sc2;
+	strappers[pid3] = &sc3;
+
+
+	std::string nulldir = "";
+
+	CacheSource *csrc1 = new CacheTestSource(nulldir);
+	CacheStore  *cstore1 = new CacheTestStore(&ctt1, nulldir);
+	CacheId cid1(TESTID, 0);
+
+	CacheSource *csrc2 = new CacheTestSource(nulldir);
+	CacheStore  *cstore2 = new CacheTestStore(&ctt2, nulldir);
+	CacheId cid2(TESTID, 0);
+
+	CacheSource *csrc3 = new CacheTestSource(nulldir);
+	CacheStore  *cstore3 = new CacheTestStore(&ctt3, nulldir);
+	CacheId cid3(TESTID, 0);
+
+	CachePair cp1(csrc1, cstore1, cid1);
+	CachePair cp2(csrc2, cstore2, cid2);
+	CachePair cp3(csrc3, cstore3, cid3);
+
+	sc1.addCachePair(cp1);
+	sc2.addCachePair(cp2);
+	sc3.addCachePair(cp3);
+
+
+	sc1.addPeerId(pid2);
+	sc2.addPeerId(pid1);
+	sc2.addPeerId(pid3);
+	sc3.addPeerId(pid2);
+
+	/* add in a cache to sc2 */
+	CacheData cdata;
+
+	cdata.pid = pid1;
+	cdata.cid = cid1;
+	cdata.name = "Perm Cache";
+	cdata.path = "./";
+	cdata.hash = "GHJKI";
+
+	csrc1->refreshCache(cdata);
+
+	cdata.pid = pid2;
+	cdata.cid = cid2;
+	cdata.name = "Funny Cache";
+	cdata.path = "./";
+	cdata.hash = "ABCDEF";
+
+	csrc2->refreshCache(cdata);
+
+	/* now exercise it */
+
+	for(int i = 0; 1 ; i++)
+	{
+		RsPeerId src("");
+		CacheStrapper *csp = NULL;
+
+		if (i % 5 == 1)
+		{
+			src = pid1;
+			csp = &sc1;
+		}
+		else if (i % 5 == 2)
+		{
+			src = pid2;
+			csp = &sc2;
+		}
+		else if (i % 5 == 3)
+		{
+			src = pid3;
+			csp = &sc3;
+		}
+		std::cerr << std::endl;
+		std::cerr << "Cache Iteraton: " << time(NULL) << std::endl;
+		std::cerr << std::endl;
+
+		if (src != "")
+		{
+			handleQuery(csp, src, strappers);
+		}
+
+
+		if (i % 21 == 0)
+		{
+			/* print out the resources */
+			sc1.listCaches(std::cerr);
+			sc2.listCaches(std::cerr);
+			sc3.listCaches(std::cerr);
+		}
+
+		/* every once in a while change the cache on 2 */
+		if (i % 31 == 25)
+		{
+			cdata.hash += "X";
+			csrc2->refreshCache(cdata);
+		}
+
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+#ifndef WINDOWS_SYS
+		sleep(1);
+#else
+		Sleep(1000);
+#endif
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+
+		/* tick the systems */
+
+	}
+
+	/* Cleanup - TODO */
+
+	return 1;
+}
+
+void handleQuery(CacheStrapper *csp, RsPeerId pid, 
+		std::map<RsPeerId, CacheStrapper *> &strappers)
+{
+	/* query */
+	std::list<RsPeerId> ids;
+	std::list<RsPeerId>::iterator pit;
+
+	std::cerr << "Cache Query from: " << pid << std::endl;
+
+	csp -> sendCacheQuery(ids, time(NULL));
+	for(pit = ids.begin(); pit != ids.end(); pit++)
+	{
+		std::cerr << "Cache Query for: " << (*pit) << std::endl;
+		std::map<RsPeerId, CacheStrapper *>::iterator sit;
+		if (strappers.end() != (sit = strappers.find(*pit)))
+		{
+			std::map<CacheId, CacheData> hashs;
+			std::map<CacheId, CacheData>::iterator hit;
+			(sit -> second) -> handleCacheQuery(pid, hashs);
+			for(hit = hashs.begin(); hit != hashs.end(); hit++)
+			{
+				csp -> recvCacheResponse(hit->second, time(NULL));
+			}
+		}
+		else
+		{
+			std::cerr << "Unknown Query Destination!" << std::endl;
+		}
+	}
+}
+

libretroshare/src/dbase/filedex.cc

@@ -0,0 +1,366 @@
+/*
+ * "$Id: filedex.cc,v 1.8 2007-02-18 21:46:49 rmf24 Exp $"
+ *
+ * Other Bits for RetroShare.
+ *
+ * Copyright 2004-2006 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+
+
+
+#include "filedex.h"
+
+#include <iostream>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+
+fdex::fdex()
+{
+	return;
+}
+
+fdex::fdex(const char *p, const char *d, const char *f, const char *e, int l, int v)
+	:path(p), subdir(d), file(f), ext(e), len(l), vis(v)
+{
+	return;
+}
+
+
+int	filedex::load(std::list<DirItem> dirs)
+{
+	std::list<DirItem>::iterator it;
+
+	/* these must be done in the correct order to ensure
+	 * that the visibility is correct
+	 */
+	for(int i = FD_VIS_EXACT_ONLY; i <= FD_VIS_LISTING; i++)
+	{
+		for(it = dirs.begin(); it != dirs.end(); it++)
+		{
+			if (i == it -> vis)
+			{
+				//std::cerr << "Adding Type(" << it -> vis << ") " << it -> basepath;
+				//std::cerr << std::endl;
+
+				dirtodo.push_back(*it);
+			}
+		}
+	}
+	return processdirs();
+}
+
+int	filedex::clear()
+{
+	std::list<fdex *>::iterator it;
+	dirtodo.clear();
+	dirdone.clear();
+
+	for(it = files.begin(); it != files.end(); it++)
+	{
+		delete(*it);
+	}
+	files.clear();
+
+	for(it = dirlist.begin(); it != dirlist.end(); it++)
+	{
+		delete(*it);
+	}
+	dirlist.clear();
+	return 1;
+}
+
+std::string strtolower2(std::string in)
+{
+	std::string out = in;
+	for(int i = 0; i < (signed) out.length(); i++)
+	{
+		if (isupper(out[i]))
+		{
+			out[i] +=  'a' - 'A';
+		}
+	}
+	return out;
+}
+
+std::string fileextension2(std::string in)
+{
+	std::string out;
+	bool found = false;
+	for(int i = in.length() -1; (i >= 0) && (found == false); i--)
+	{
+		if (in[i] == '.')
+		{
+			found = true;
+			for(int j = i+1; j < (signed) in.length(); j++)
+			{
+				out += in[j];
+			}
+		}
+	}
+	return strtolower2(out);
+}
+
+std::list<fdex *> filedex::search(std::list<std::string> terms, int limit, bool local)
+{
+	bool found;
+	std::list<fdex *> newlist;
+	int matches = 0;
+
+	std::list<fdex *>::iterator fit;
+	std::list<std::string>::iterator tit;
+
+	//std::cerr << "filedex::search() looking for" << std::endl;
+	//for(tit = terms.begin(); tit != terms.end(); tit++)
+	//	std::cerr << "\t(" << *tit << ")" << std::endl;
+
+	//std::cerr << "Checking:" << std::endl;
+
+	for(fit = files.begin(); fit != files.end(); fit++)
+	{
+		// ignore named only files.
+		if ((!local) && ((*fit)->vis < FD_VIS_SEARCH))
+			continue;
+		found = true;
+		for(tit = terms.begin(); (tit != terms.end()) && (found); tit++)
+		{
+			std::string path = (*fit) -> subdir+"/"+(*fit)->file;
+			std::string lowerpath = strtolower2(path);
+			std::string lowerterm = strtolower2(*tit);
+
+			//std::cerr << "\tLooking for (" << lowerterm;
+			//std::cerr << ") in (" << lowerpath << ")  ";
+
+			if (strstr(lowerpath.c_str(), lowerterm.c_str()) == NULL)
+			{
+				found = false;
+				//std::cerr << "\tFalse..." << std::endl;
+			}
+			else
+			{
+				//std::cerr << "\tTrue..." << std::endl;
+			}
+		}
+		if (found)
+		{
+			//std::cerr << "Found Matching!" << std::endl;
+			newlist.push_back(*fit);
+			if (++matches == limit)
+			{
+				//std::cerr << "Reached Limit(" << limit << ")";
+				//std::cerr << "Returning Results" << std::endl;
+				return newlist;
+			}
+		}
+	}
+	return newlist;
+}
+
+
+std::list<fdex *> filedex::dirlisting(std::string basedir)
+{
+	std::list<fdex *> newlist;
+	std::list<fdex *>::iterator fit;
+
+	//std::cerr << "filedex::dirlisting() looking for subdir: " << basedir << std::endl;
+
+	//std::cerr << "Checking:" << std::endl;
+
+	for(fit = dirlist.begin(); fit != dirlist.end(); fit++)
+	{
+		//std::cerr << "DCHK(" << basedir << ") vs (" << (*fit) -> subdir << ")" << std::endl;
+		if (basedir == (*fit) -> subdir)
+		{
+			/* in the dir */
+			//std::cerr << "Found Matching SubDir:";
+			//std::cerr << (*fit) -> subdir << std::endl;
+			newlist.push_back(*fit);
+		}
+	}
+
+	for(fit = files.begin(); fit != files.end(); fit++)
+	{
+		//std::cerr << "FCHK(" << basedir << ") vs (" << (*fit) -> subdir << ")" << std::endl;
+		if (basedir == (*fit) -> subdir)
+		{
+			/* in the dir */
+			//std::cerr << "Found Matching File:";
+			//std::cerr << (*fit) -> subdir << std::endl;
+			newlist.push_back(*fit);
+		}
+	}
+	return newlist;
+}
+
+
+std::list<fdex *> filedex::findfilename(std::string name)
+{
+	std::list<fdex *> newlist;
+	std::list<fdex *>::iterator fit;
+
+	std::cerr << "filedex::findfilename() looking for: " << name << std::endl;
+
+	std::cerr << "Checking:" << std::endl;
+	for(fit = files.begin(); fit != files.end(); fit++)
+	{
+		if (name == (*fit) -> file)
+		{
+			/* in the dir */
+			std::cerr << "Found Matching File!" << std::endl;
+			newlist.push_back(*fit);
+		}
+	}
+	return newlist;
+}
+
+
+int	filedex::processdirs()
+{
+	std::list<DirItem>::iterator it;
+	std::list<std::string>::iterator sit;
+	std::string dirname;
+	std::string subdir;
+	std::string fname;
+	std::string fullname;
+	struct dirent *ent;
+	struct stat buf;
+	bool found = false;
+	fdex *fx;
+	int count = 0;
+	int ficount = 0;
+	while(dirtodo.size() > 0)
+	{
+		count++;
+		it = dirtodo.begin();
+		DirItem currDir(*it);
+		dirname = currDir.getFullPath();
+		dirtodo.erase(it);
+
+		//std::cerr << "\tExamining: " << currDir.basepath;
+		//std::cerr << " -/- " << currDir.subdir << std::endl;
+		//std::cerr << "\t\t" <<  count << " Directories done, ";
+		//std::cerr << dirtodo.size() << " to go! " << std::endl;
+
+		// open directory.
+		DIR *dir = opendir(dirname.c_str());
+
+		if (dir != NULL) {
+		// read the directory.
+		while(NULL != (ent = readdir(dir)))
+		{
+			fname = ent -> d_name;
+			fullname = dirname + "/" + fname;
+			subdir  = currDir.subdir;
+		//	std::cerr << "Statting dirent: " << fullname.c_str() <<std::endl;
+		//	std::cerr << "Statting dirent: " << fullname <<std::endl;
+
+			// stat each file.
+			if (-1 != stat(fullname.c_str(), &buf))
+			{
+		//std::cerr << "buf.st_mode: " << buf.st_mode <<std::endl;
+				if (S_ISDIR(buf.st_mode))
+				{
+		//std::cerr << "Is Directory: " << fullname << std::endl;
+				  found = false;
+				  // remove "." and ".." entries.
+				  if ((fname == ".") || (fname == ".."))
+				  { 
+					  found = true;
+				  }
+
+				  // check if in list of done already.
+				  for(it = dirtodo.begin(); it != dirtodo.end(); it++)
+				  {
+					if (it -> getFullPath() == fullname)
+						found = true;
+				  }
+				  for(sit = dirdone.begin(); sit != dirdone.end(); sit++)
+				  {
+					if ((*sit) == fullname)
+						found = true;
+				  }
+				  if (found == false)
+				  {
+					// add to list to read.
+					DirItem ndir(currDir.basepath, subdir+"/"+fname, currDir.vis);
+					// Push to the front of the list -> so processing done in order.
+					dirtodo.push_front(ndir);
+		//			std::cerr << "\t Adding Directory: " << fullname;
+		//			std::cerr << " to dirtodo" << std::endl;
+
+					if (currDir.vis == FD_VIS_LISTING)
+					{
+						// add to dirlist dbase.
+						fx = new fdex(fullname.c_str(), subdir.c_str(), 
+						fname.c_str(), "DIR", 0, currDir.vis);
+						dirlist.push_back(fx);
+		//				std::cerr << "\t Adding Directory: " << fullname;
+		//				std::cerr << " to dirlist" << std::endl;
+					}
+				  }
+				}
+				else if (S_ISREG(buf.st_mode))
+				{
+					// add to dbase.
+					fx = new fdex(fullname.c_str(), subdir.c_str(), 
+						fname.c_str(), (fileextension2(fname)).c_str(), 
+						buf.st_size, currDir.vis);
+
+					//fx -> path = fullname;
+					files.push_back(fx);
+					ficount++;
+					if (ficount % 100 == 0)
+					{
+				//std::cerr << "\tIndex(" << ficount << ") : ";
+				//std::cerr << "\tIndexing File: " << fname;
+				//std::cerr << std::endl;
+					}
+
+				}
+				else
+				{
+					// warning.
+					std::cerr << "\t Ignoring Unknown FileType";
+					std::cerr << std::endl;
+				}
+			}
+			else
+			{
+
+			}
+
+		}
+		// should check error message.
+		dirdone.push_back(dirname);
+		closedir(dir);
+		} 
+		else
+		{
+			//std::cerr << "Cannot Open Directory:" << dirname << std::endl;
+		}
+
+	}
+	return 1;
+}
+
+

libretroshare/src/dbase/filedex.h

@@ -0,0 +1,111 @@
+/*
+ * "$Id: filedex.h,v 1.4 2007-02-18 21:46:49 rmf24 Exp $"
+ *
+ * Other Bits for RetroShare.
+ *
+ * Copyright 2004-2006 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+
+
+#ifndef MRK_PQI_FILEINDEXER
+#define MRK_PQI_FILEINDEXER
+
+/* a file index.
+ *
+ * initiated with a list of directories.
+ *
+ */
+
+#include <list>
+#include <string>
+
+#define FD_VIS_EXACT_ONLY    0
+#define FD_VIS_SEARCH        1
+#define FD_VIS_LISTING       2
+
+/* if vis = FD_VIS_EXACT_ONLY 
+ * Then only an exact filename match works
+ * if vis = FD_VIS_SEARCH
+ * Then only a search will return it.
+ * if vis = FD_VIS_LISTING
+ * Then any method will find it.
+ */
+
+class fdex
+{
+	public:
+	fdex();
+	fdex(const char *path, const char *srchpath, 
+		const char *file, const char *ext, int len, int vis);
+
+std::string path; // full path. (not searched)
+std::string subdir; // searched if flag set.
+std::string file;
+std::string ext;
+int	len;
+int     vis; 
+
+};
+
+
+class DirItem
+{
+	public:
+	DirItem(std::string bp, std::string sd, int v)
+	: basepath(bp), subdir(sd), vis(v) {}
+	std::string	getFullPath()
+	{
+		return basepath + subdir;
+	}
+	std::string basepath;
+	std::string subdir;
+	int vis;
+};
+
+class filedex
+{
+	public:
+
+int	load(std::list<DirItem> dirs);
+int	clear();
+
+// -1 for no limit.
+// If remote ... then restrictions apply.
+std::list<fdex *> search(std::list<std::string>, int limit, bool local); 
+std::list<fdex *> dirlisting(std::string); 
+std::list<fdex *> findfilename(std::string); 
+
+	private:
+
+int	processdirs();
+
+	std::list<DirItem> dirtodo;
+	std::list<std::string> dirdone;
+
+	/* dbase */
+	std::list<fdex *> files;
+	std::list<fdex *> dirlist;
+};
+
+
+
+
+#endif

libretroshare/src/dbase/filelook.cc

@@ -0,0 +1,538 @@
+/*
+ * "$Id: filelook.cc,v 1.3 2007-03-05 21:26:03 rmf24 Exp $"
+ *
+ * Other Bits for RetroShare.
+ *
+ * Copyright 2004-2006 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+
+
+#include "dbase/filelook.h"
+#include "util/rsdir.h"
+
+#include <sstream>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <dirent.h>
+
+
+/* 
+ * The Basic Low-Level Local File/Directory Interface.
+ *
+ * Requests are queued, and serviced (slowly)
+ *
+ *
+ * So this is a simple
+ * Local Lookup service,
+ * with a locked threaded interface.
+ *
+ * Input:
+ *    "lookup directory".
+ *
+ * Output:
+ *    "FileItems"
+ *    "DirItems"
+ *
+ *
+ * a file index.
+ *
+ * initiated with a list of directories.
+ *
+ */
+
+
+fileLook::fileLook()
+{
+	return;
+}
+
+fileLook::~fileLook()
+{
+
+	return;
+}
+
+	/* interface */
+void	fileLook::setSharedDirectories(std::list<std::string> dirs)
+{
+	int i;
+	lockDirs(); { /* LOCKED DIRS */
+
+	/* clear old directories */
+	sharedDirs.clear();
+
+	/* iterate through the directories */
+	std::list<std::string>::iterator it;
+	for(it = dirs.begin(); it != dirs.end(); it++)
+	{
+		/* get the head directory */
+		std::string root_dir = *it;
+		std::string top_dir  = RsDirUtil::getTopDir(root_dir);
+
+		/* if unique -> add, else add modifier  */
+		bool unique = false;
+		for(i = 0; !unique; i++)
+		{
+			std::string tst_dir = top_dir;
+			if (i > 0)
+			{
+				std::ostringstream out;
+				out << "-" << i;
+				tst_dir += out.str();
+			}
+			std::map<std::string, std::string>::const_iterator cit;
+			if (sharedDirs.end()== (cit=sharedDirs.find(tst_dir)))
+			{
+				unique = true;
+				/* add it! */
+				sharedDirs[tst_dir.c_str()] = root_dir;
+				std::cerr << "Added [" << tst_dir << "] => " << root_dir << std::endl;
+			}
+		}
+	}
+
+	} unlockDirs(); /* UNLOCKED DIRS */
+}
+
+
+
+int	fileLook::lookUpDirectory(PQItem *i)
+{
+	std::cerr << "lookUpDirectory() About to Lock" << std::endl;
+	lockQueues();
+
+	std::cerr << "lookUpDirectory():" << std::endl;
+	i -> print(std::cerr);
+
+	mIncoming.push_back(i);
+
+	unlockQueues();
+	std::cerr << "lookUpDirectory() Unlocked" << std::endl;
+
+	return 1;
+}
+
+PQItem  *fileLook::getResult()
+{
+	PQItem *i = NULL;
+
+	lockQueues();
+
+	if (mOutgoing.size() > 0)
+	{
+		i = mOutgoing.front();
+		mOutgoing.pop_front();
+
+		std::cerr << "getResult()" << std::endl;
+		//i -> print(std::cerr);
+	}
+
+	unlockQueues();
+
+	return i;
+}
+
+	/* locking 
+void	fileLook::lockQueues() 
+{}
+
+void	fileLook::unlockQueues() 
+{}
+
+void	fileLook::lockDirs() 
+{}
+
+void	fileLook::unlockDirs() 
+{}
+
+************/
+
+void	fileLook::run()
+{
+	std::cerr << "fileLook::run() started." << std::endl;
+	processQueue();
+}
+
+void	fileLook::processQueue()
+{
+	while(1)
+	{
+		
+		PQItem *i = NULL;
+
+		//std::cerr << "fileLook::run() -> lockQueues()" << std::endl;
+		lockQueues();
+
+		if (mIncoming.size() > 0)
+		{
+			i = mIncoming.front();
+			mIncoming.pop_front();
+		}
+
+		unlockQueues();
+		//std::cerr << "fileLook::run() -> unlockQueues()" << std::endl;
+
+		if (i)
+		{
+			/* process item */
+
+			/* look up the directory */
+			std::cerr << "fileLook::run() -> loadDirectory()" << std::endl;
+			loadDirectory(i);
+		}
+		else
+		{
+			/* sleep */
+			//std::cerr << "fileLook::sleep() ;)" << std::endl;
+#ifndef WINDOWS_SYS /* UNIX */
+			usleep(50000); /* 50 milli secs (1/20)th of sec */
+#else
+			Sleep(50); /* 50 milli secs (1/20)th of sec */
+#endif
+
+		}
+	}
+
+}
+
+
+/* THIS IS THE ONLY BIT WHICH DEPENDS ON PQITEM STRUCTURE */
+void	fileLook::loadDirectory(PQItem *dir)
+{
+	/* extract the path */
+	std::cerr << "loadDirectory()" << std::endl;
+
+	/* check its a PQFileItem */
+	PQFileItem *fi = dynamic_cast<PQFileItem *>(dir);
+	if (!fi)
+	{
+		return;
+	}
+
+	/* we need the root directory, and the rest */
+	std::string req_dir = fi -> path;
+	std::string root_dir  = RsDirUtil::getRootDir(req_dir);
+	std::string rest_path  = RsDirUtil::removeRootDirs(req_dir, root_dir);
+
+	/* get directory listing */
+	std::string full_root = "/dev/null";
+
+	if (root_dir == "")
+	{
+		std::cerr << "loadDirectory() Root Request" << std::endl;
+		loadRootDirs(fi);
+	}
+
+	int err = 0;
+
+	lockDirs();
+	{
+		/* check that dir is one of the shared */
+		std::map<std::string, std::string>::iterator it;
+		it = sharedDirs.find(root_dir.c_str());
+		if (it == sharedDirs.end())
+		{
+			err = 1;
+		}
+		else
+		{
+			full_root = it -> second;
+		}
+	}
+	unlockDirs();
+	std::cerr << "root_dir = [" << root_dir << "]" << std::endl;
+	std::cerr << "full_root = " << full_root << std::endl;
+
+	if (err)
+	{
+		std::cerr << "Rejected: root_dir = " << root_dir << std::endl;
+		for(unsigned int i = 0; i < root_dir.length(); i++)
+		{
+			std::cerr << "[" << (int) root_dir[i] << "]:[" << (int) root_dir[i] << "]" << std::endl;
+		}
+			
+		std::map<std::string, std::string>::iterator it;
+		for(it = sharedDirs.begin(); it != sharedDirs.end(); it++)
+		{
+			std::cerr << "Possible Root: " << it -> first << std::endl;
+		}
+
+		return;
+	}
+
+	/* otherwise load directory */
+	std::string full_path = full_root;
+
+	if (rest_path.length() > 0)
+	{
+		full_path += "/" + rest_path;
+	}
+
+	std::cerr << "Looking up path:" << full_path << std::endl;
+
+	/* lookup base dir */
+        DIR *rootdir = opendir(full_path.c_str());
+	struct dirent *rootent, *subent;
+	struct stat buf;
+
+        if (rootdir == NULL) {
+		/* error */
+		std::cerr << "Error in Path" << std::endl;
+		return;
+	}
+
+	std::list<PQItem *> subitems;
+	int rootcount = 0;
+        while(NULL != (rootent = readdir(rootdir)))
+	{
+		/* iterate through the entries */
+		bool issubdir = false;
+
+		/* check entry type */
+		std::string fname = rootent -> d_name;
+		std::string fullname = full_path + "/" + fname;
+		std::cerr << "Statting dirent: " << fullname <<std::endl;
+
+		if (-1 != stat(fullname.c_str(), &buf))
+		{
+			std::cerr << "buf.st_mode: " << buf.st_mode <<std::endl;
+			if (S_ISDIR(buf.st_mode))
+			{
+				std::cerr << "Is Directory: " << fullname << std::endl;
+				if ((fname == ".") || (fname == ".."))
+				{
+					std::cerr << "Skipping:" << fname << std::endl;
+					continue; /* skipping links */
+				}
+				issubdir = true;
+			}
+			else if (S_ISREG(buf.st_mode))
+			{
+				/* is file */
+				std::cerr << "Is File: " << fullname << std::endl;
+				issubdir = false;
+			}
+			else
+			{
+				/* unknown , ignore */
+				continue;
+			}
+		}
+
+
+		/* If subdir - Count the subentries */
+		DIR *subdir = NULL;
+		if (issubdir)
+		{
+			subdir = opendir(fullname.c_str());
+		}
+		int subdirno = 0;
+		if (subdir)
+		{
+        		while(NULL != (subent = readdir(subdir)))
+			{
+
+                        	std::string fname = subent -> d_name;
+				if (
+#ifndef WINDOWS_SYS /* UNIX */
+				   (DT_DIR == subent -> d_type) &&  /* Only works under Unix */
+#else /* WIndows */
+#endif
+					((fname == ".") || (fname == "..")))
+				{
+					/* do not add! */
+					std::cerr << "Removed entry:" << fname << " from subdir count" << std::endl;
+				}
+				else
+				{
+					subdirno++;
+				}
+			}
+			/* clean up dir */
+        		closedir(subdir);
+		}
+		rootcount++;
+
+		std::string subname = rootent -> d_name;
+
+		/* create an item */
+		PQFileItem *finew = fi -> clone();
+
+		// path should be copied auto.
+		finew -> name = subname;
+
+		if (issubdir)
+		{
+			finew -> reqtype = PQIFILE_DIRLIST_DIR;
+			finew -> size = subdirno; /* number of entries in dir */
+		}
+		else
+		{
+			finew -> reqtype = PQIFILE_DIRLIST_FILE;
+			finew -> size = buf.st_size;
+		}
+		subitems.push_back(finew);
+	}
+
+	/* clean up dir */
+        closedir(rootdir);
+
+
+	/* if necessary create Root Entry */
+
+
+	/* if constructing Tree - do it here */
+
+	/* put on queue */
+	lockQueues();
+
+	std::list<PQItem *>::iterator it;
+	for(it = subitems.begin(); it != subitems.end(); it++)
+	{
+		/* push onto outgoing queue */
+		mOutgoing.push_back(*it);
+	}
+	subitems.clear();
+
+	unlockQueues();
+};
+
+void	fileLook::loadRootDirs(PQFileItem *dir)
+{
+	/* extract the path */
+	std::cerr << "loadRootDirs()" << std::endl;
+
+	dir -> path = "";
+	int rootcount = 0;
+
+	lockDirs();
+
+	std::list<PQItem *> subitems;
+	std::map<std::string, std::string>::iterator it;
+	for(it = sharedDirs.begin(); it != sharedDirs.end(); it++)
+	{
+
+		/* lookup base dir */
+        	DIR *entdir = opendir(it -> second.c_str());
+		struct dirent *subent;
+
+		if (!entdir)
+		{
+			std::cerr << "Bad Shared Dir: " << it->second << std::endl;
+			continue;
+		}
+
+		int subdirno = 0;
+        	while(NULL != (subent = readdir(entdir)))
+		{
+
+                       	std::string fname = subent -> d_name;
+			if (
+#ifndef WINDOWS_SYS /* UNIX */
+				   (DT_DIR == subent -> d_type) &&  /* Only works under Unix */
+#else /* WIndows */
+#endif
+				((fname == ".") || (fname == "..")))
+			{
+				/* do not add! */
+				std::cerr << "Removed entry:" << fname << " from subdir count" << std::endl;
+			}
+			else
+			{
+				subdirno++;
+			}
+		}
+
+		closedir(entdir);
+
+		rootcount++;
+
+		/* create an item */
+		PQFileItem *finew = dir -> clone();
+
+		// path should be copied auto.
+		finew -> name = it -> first;
+		finew -> reqtype = PQIFILE_DIRLIST_DIR;
+		finew -> size = subdirno; /* number of entries in dir */
+
+		subitems.push_back(finew);
+	}
+	unlockDirs();
+
+
+	/* put on queue */
+	lockQueues();
+
+	std::list<PQItem *>::iterator it2;
+	for(it2 = subitems.begin(); it2 != subitems.end(); it2++)
+	{
+		/* push onto outgoing queue */
+		mOutgoing.push_back(*it2);
+	}
+	subitems.clear();
+
+	unlockQueues();
+};
+
+
+PQFileItem *fileLook::findFileEntry(PQFileItem *fi)
+{
+	/* extract the path */
+	std::cerr << "findFileEntry()" << std::endl;
+
+	/* we need the root directory, and the rest */
+	std::string req_dir = fi -> path;
+	std::string root_dir  = RsDirUtil::getRootDir(req_dir);
+	std::string rest_path  = RsDirUtil::removeRootDirs(req_dir, root_dir);
+
+	/* get directory listing */
+	std::string full_root = "/dev/null";
+
+	int err = 0;
+
+	lockDirs();
+	{
+		/* check that dir is one of the shared */
+		std::map<std::string, std::string>::iterator it;
+		it = sharedDirs.find(root_dir.c_str());
+		if (it == sharedDirs.end())
+		{
+			err = 1;
+		}
+		else
+		{
+			full_root = it -> second;
+		}
+	}
+	unlockDirs();
+
+	PQFileItem *rtn = NULL;
+	if (err)
+		return rtn;
+	
+	rtn = fi -> clone();
+	rtn -> path = full_root + "/" + rest_path;
+
+	return rtn;
+}
+

libretroshare/src/dbase/filelook.h

@@ -0,0 +1,113 @@
+/*
+ * "$Id: filelook.h,v 1.1 2007-02-19 20:08:30 rmf24 Exp $"
+ *
+ * Other Bits for RetroShare.
+ *
+ * Copyright 2004-2006 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+
+#ifndef MRK_PQI_FILELOOK
+#define MRK_PQI_FILELOOK
+
+/* 
+ * The Basic Low-Level Local File/Directory Interface.
+ *
+ * Requests are queued, and serviced (slowly)
+ *
+ *
+ * So this is a simple
+ * Local Lookup service,
+ * with a locked threaded interface.
+ *
+ * Input:
+ *    "lookup directory".
+ *
+ * Output:
+ *    "FileItems"
+ *    "DirItems"
+ *
+ *
+ * a file index.
+ *
+ * initiated with a list of directories.
+ *
+ */
+
+#include <list>
+#include <map>
+
+#include "pqi/pqi.h"
+#include "util/rsthreads.h"
+
+
+class fileLook: public RsThread
+{
+	public:
+
+	fileLook();
+virtual ~fileLook();
+
+	/* threading */
+virtual void	run(); /* overloaded */
+
+	/* interface */
+void	setSharedDirectories(std::list<std::string> dirs);
+
+	/* I/O */
+int	lookUpDirectory(PQItem *i);
+	/* This one goes directly -> for speed, actually just a basepath lookup (no file check) */
+PQFileItem *findFileEntry(PQFileItem *);
+
+PQItem  *getResult();
+
+	private:
+
+
+	/* locking */
+void	lockQueues()   { qMtx.lock();   }
+void	unlockQueues() { qMtx.unlock(); }
+
+void	lockDirs()   { dMtx.lock();   }
+void	unlockDirs() { dMtx.unlock(); }
+
+	RsMutex qMtx;
+	RsMutex dMtx;
+
+	/* threading queues */
+std::list<PQItem *> mIncoming;
+std::list<PQItem *> mOutgoing;
+
+std::map<std::string, std::string> sharedDirs;
+
+	/* Processing Fns */
+
+void	processQueue();
+
+/* THIS IS THE ONLY BIT WHICH DEPENDS ON PQITEM STRUCTURE */
+void	loadDirectory(PQItem *dir);
+void    loadRootDirs(PQFileItem *dir);
+
+};
+
+
+
+#endif
+

libretroshare/src/dbase/fimonitor.cc

@@ -0,0 +1,672 @@
+/*
+ * RetroShare FileCache Module: fimonitor.cc
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "dbase/fimonitor.h"
+#include "util/rsdir.h"
+#include <iostream>
+#include <sstream>
+#include <iomanip>
+
+#include <dirent.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <openssl/sha.h>
+
+#define FIM_DEBUG 1
+
+FileIndexMonitor::FileIndexMonitor(std::string cachedir, std::string pid)
+	:CacheSource(CACHE_TYPE_FILE_INDEX, false, cachedir), fi(pid), 
+		pendingDirs(false), pendingForceCacheWrite(false)
+
+{
+	updatePeriod = 60;
+}
+
+
+FileIndexMonitor::~FileIndexMonitor()
+{
+	/* Data cleanup - TODO */
+	return;
+}
+
+bool    FileIndexMonitor::findLocalFile(std::string hash, 
+				std::string &fullpath, uint32_t &size)
+{
+	std::list<FileEntry *> results;
+	bool ok = false;
+
+	fiMutex.lock(); { /* LOCKED DIRS */
+
+	std::cerr << "FileIndexMonitor::findLocalFile() Hash: " << hash << std::endl;
+	/* search through the fileIndex */
+	fi.searchHash(hash, results);
+	if (results.size() > 0)
+	{
+		/* find the full path for the first entry */
+		FileEntry *fe = results.front();
+		DirEntry  *de = fe->parent; /* all files must have a valid parent! */
+
+		std::cerr << "FileIndexMonitor::findLocalFile() Found Name: " << fe->name << std::endl;
+		std::string shpath =  RsDirUtil::removeRootDir(de->path);
+		std::string basedir = RsDirUtil::getRootDir(de->path);
+		std::string realroot = findRealRoot(basedir);
+
+		/* construct full name */
+		if (realroot.length() > 0)
+		{
+			fullpath = realroot + "/";
+			if (shpath != "")
+			{
+				fullpath += shpath + "/";
+			}
+			fullpath += fe->name;
+
+			size = fe->size;
+			ok = true;
+		}
+		std::cerr << "FileIndexMonitor::findLocalFile() Found Path: " << fullpath << std::endl;
+		std::cerr << "FileIndexMonitor::findLocalFile() Found Size: " << size << std::endl;
+	}
+
+
+	} fiMutex.unlock(); /* UNLOCKED DIRS */
+
+	return ok;
+}
+
+
+bool FileIndexMonitor::loadCache(const CacheData &data)  /* called with stored data */
+{
+	bool ok = false;
+
+	fiMutex.lock(); { /* LOCKED DIRS */
+
+	//fi.root->name = data.pid;
+
+	/* More error checking needed here! */
+	if ((ok = fi.loadIndex(data.path + '/' + data.name, 
+				data.hash, data.size)))
+	{
+		std::cerr << "FileIndexMonitor::loadCache() Success!";
+		std::cerr << std::endl;
+	}
+	else
+	{
+		std::cerr << "FileIndexMonitor::loadCache() Failed!";
+		std::cerr << std::endl;
+	}
+		
+	} fiMutex.unlock(); /* UNLOCKED DIRS */
+
+	if (ok)
+	{
+		return updateCache(data);
+	}
+	return false;
+}
+
+bool FileIndexMonitor::updateCache(const CacheData &data)  /* we call this one */
+{
+	return refreshCache(data);
+}
+
+
+void 	FileIndexMonitor::setPeriod(int period)
+{
+	updatePeriod = period;
+}
+
+void 	FileIndexMonitor::run()
+{
+
+	updateCycle();
+
+	while(1)
+	{
+
+		for(int i = 0; i < updatePeriod; i++)
+		{
+
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+#ifndef WINDOWS_SYS
+			sleep(1);
+#else
+
+                	Sleep(1000);
+#endif
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+
+			/* check dirs if they've changed */
+			if (internal_setSharedDirectories())
+			{
+				break;
+			}
+		}
+
+		updateCycle();
+	}
+}
+
+
+void 	FileIndexMonitor::updateCycle()
+{
+	time_t startstamp = time(NULL);
+
+	/* iterate through all out-of-date directories */
+	bool moretodo = true;
+	bool fiMods = false;
+
+	while(moretodo)
+	{
+		/* sleep a bit for each loop */
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+#ifndef WINDOWS_SYS
+		usleep(100000); /* 1/10 sec */
+#else
+
+                Sleep(100);
+#endif
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+
+		/* check if directories have been updated */
+		if (internal_setSharedDirectories())
+		{
+			/* reset start time */
+			startstamp = time(NULL);
+		}
+		
+		/* Handle a Single out-of-date directory */
+
+		time_t stamp = time(NULL);
+
+		/* lock dirs */
+		fiMutex.lock();
+
+        	DirEntry *olddir = fi.findOldDirectory(startstamp);
+
+		if (!olddir)
+		{
+			/* finished */
+			fiMutex.unlock();
+			moretodo = false;
+			continue;
+		}
+
+#ifdef FIM_DEBUG
+		std::cerr << "FileIndexMonitor::updateCycle()";
+		std::cerr << " Checking: " << olddir->path << std::endl;
+#endif
+
+
+        	FileEntry fe;
+		/* entries that need to be checked properly */
+		std::list<FileEntry> filesToHash; 
+		std::list<FileEntry>::iterator hit;
+
+		/* determine the full root path */
+		std::string dirpath = olddir->path;
+		std::string rootdir = RsDirUtil::getRootDir(olddir->path);
+		std::string remdir  = RsDirUtil::removeRootDir(olddir->path);
+
+		std::string realroot = findRealRoot(rootdir);
+
+		std::string realpath = realroot;
+		if (remdir != "")
+		{
+			realpath += "/" + remdir;
+		}
+			
+
+#ifdef FIM_DEBUG
+		std::cerr << "FileIndexMonitor::updateCycle()";
+		std::cerr << " RealPath: " << realpath << std::endl;
+#endif
+
+		/* check for the dir existance */
+		DIR *dir = opendir(realpath.c_str());
+		if (!dir)
+		{
+			std::cerr << "FileIndexMonitor::updateCycle()";
+			std::cerr << " Missing Dir: " << realpath << std::endl;
+			/* bad directory - delete */
+                	if (!fi.removeOldDirectory(olddir->parent->path, olddir->name, stamp))
+			{
+				/* bad... drop out of updateCycle() - hopefully the initial cleanup
+				 * will deal with it next time! - otherwise we're in a continual loop
+				 */
+				std::cerr << "FileIndexMonitor::updateCycle()";
+                        	std::cerr << "ERROR Failed to Remove: " << olddir->path << std::endl;
+			}
+
+			fiMutex.unlock();
+			continue;
+		}
+
+                /* update this dir - as its valid */
+                fe.name = olddir->name;
+                fi.updateDirEntry(olddir->parent->path, fe, stamp);
+
+                /* update the directories and files here */
+                std::map<std::string, DirEntry *>::iterator  dit;
+                std::map<std::string, FileEntry *>::iterator fit;
+
+		/* flag existing subdirs as old */
+                for(dit = olddir->subdirs.begin(); dit != olddir->subdirs.end(); dit++)
+                {
+                        fe.name = (dit->second)->name;
+                        /* set the age as out-of-date so that it gets checked */
+                        fi.updateDirEntry(olddir->path, fe, 0);
+                }
+
+		/* now iterate through the directory...
+		 * directories - flags as old,
+		 * files checked to see if they have changed. (rehashed)
+		 */
+
+		struct dirent *dent;
+		struct stat buf;
+
+		while(NULL != (dent = readdir(dir)))
+		{
+			/* check entry type */
+			std::string fname = dent -> d_name;
+			std::string fullname = realpath + "/" + fname;
+
+	 		if (-1 != stat(fullname.c_str(), &buf))
+			{
+#ifdef FIM_DEBUG
+				std::cerr << "buf.st_mode: " << buf.st_mode <<std::endl;
+#endif
+	 			if (S_ISDIR(buf.st_mode))
+				{
+					if ((fname == ".") || (fname == ".."))
+					{
+#ifdef FIM_DEBUG
+						std::cerr << "Skipping:" << fname << std::endl;
+#endif
+						continue; /* skipping links */
+					}
+
+#ifdef FIM_DEBUG
+	 				std::cerr << "Is Directory: " << fullname << std::endl;
+#endif
+
+					/* add in directory */
+                        		fe.name = fname;
+                        		/* set the age as out-of-date so that it gets checked */
+                        		fi.updateDirEntry(olddir->path, fe, 0);
+				}
+				else if (S_ISREG(buf.st_mode))
+				{
+					/* is file */
+					bool toadd = false;
+#ifdef FIM_DEBUG
+					std::cerr << "Is File: " << fullname << std::endl;
+#endif
+
+                        		fe.name = fname;
+					fe.size = buf.st_size;
+					fe.modtime = buf.st_mtime;
+
+					/* check if it exists already */
+                			fit = olddir->files.find(fname);
+					if (fit == olddir->files.end())
+					{
+						/* needs to be added */
+#ifdef FIM_DEBUG
+						std::cerr << "File Missing from List:" << fname << std::endl;
+#endif
+						toadd = true;
+					}
+					else
+					{
+						/* check size / modtime are the same */
+						if ((fe.size != (fit->second)->size) ||
+						    (fe.modtime != (fit->second)->modtime))
+						{
+#ifdef FIM_DEBUG
+						std::cerr << "File ModTime/Size changed:" << fname << std::endl;
+#endif
+							toadd = true;
+						}
+						else
+						{
+							/* keep old info */
+							fe.hash = (fit->second)->hash;
+						}
+					}
+					if (toadd)
+					{
+						/* push onto Hash List */
+#ifdef FIM_DEBUG
+						std::cerr << "Adding to Update List: ";
+						std::cerr << olddir->path;
+						std::cerr << fname << std::endl;
+#endif
+						filesToHash.push_back(fe);
+					}
+					else
+					{
+						/* update with new time */
+#ifdef FIM_DEBUG
+						std::cerr << "File Hasn't Changed:" << fname << std::endl;
+#endif
+                        			fi.updateFileEntry(olddir->path, fe, stamp);
+					}
+	 			}
+				else
+				{
+					/* unknown , ignore */
+					continue;
+				}
+			}
+		}
+
+
+		/* now we unlock the lock, and iterate through the
+		 * next files - hashing them, before adding into the system.
+		 */
+		/* for safety - blank out data we cannot use (TODO) */
+		olddir = NULL;	
+
+		/* close directory */
+		closedir(dir);
+			
+		/* unlock dirs */
+		fiMutex.unlock();
+
+		if (filesToHash.size() > 0)
+		{
+			std::cerr << "List of Files to rehash in: " << dirpath << std::endl;
+			fiMods = true;
+		}
+
+                for(hit = filesToHash.begin(); hit != filesToHash.end(); hit++)
+		{
+			std::cerr << "\t" << hit->name << std::endl;
+		}
+
+		if (filesToHash.size() > 0)
+		{
+			std::cerr << std::endl;
+		}
+			
+                /* update files */
+                for(hit = filesToHash.begin(); hit != filesToHash.end(); hit++)
+                {
+			if (hashFile(realpath, (*hit)))
+			{
+				/* lock dirs */
+				fiMutex.lock();
+
+				/* update fileIndex with new time */
+				/* update with new time */
+                        	fi.updateFileEntry(dirpath, *hit, stamp);
+				
+				/* unlock dirs */
+				fiMutex.unlock();
+			}
+			else
+			{
+				std::cerr << "Failed to Hash File!" << std::endl;
+			}
+
+			/* don't hit the disk too hard! */
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+#ifndef WINDOWS_SYS
+			usleep(10000); /* 1/100 sec */
+#else
+
+       		        Sleep(10);
+#endif
+/********************************** WINDOWS/UNIX SPECIFIC PART ******************/
+
+                }
+        }
+
+	fiMutex.lock(); { /* LOCKED DIRS */
+
+	/* finished update cycle - cleanup extra dirs/files that 
+	 * have not had their timestamps updated.
+	 */
+
+	if (fi.cleanOldEntries(startstamp))
+	{
+		//fiMods = true;
+	}
+
+	/* print out the new directory structure */
+
+	fi.printFileIndex(std::cerr);
+
+	/* now if we have changed things -> restore file/hash it/and 
+	 * tell the CacheSource
+	 */
+
+	if (pendingForceCacheWrite)
+	{
+		pendingForceCacheWrite = false;
+		fiMods = true;
+	}
+
+	} fiMutex.unlock(); /* UNLOCKED DIRS */
+
+
+	if (fiMods)
+	{
+		/* store to the cacheDirectory */
+		fiMutex.lock(); { /* LOCKED DIRS */
+
+		std::string path = getCacheDir();
+		std::ostringstream out; 
+		out << "fc-own-" << time(NULL) << ".rsfc";
+
+		std::string tmpname = out.str();
+		std::string fname = path + "/" + tmpname;
+
+#ifdef FIM_DEBUG
+		std::cerr << "FileIndexMonitor::updateCycle() FileIndex modified ... updating";
+		std::cerr <<  std::endl;
+		std::cerr << "FileIndexMonitor::updateCycle() saving to: " << fname;
+		std::cerr <<  std::endl;
+#endif
+
+		std::string calchash;
+		uint32_t size;
+
+		fi.saveIndex(fname, calchash, size);
+
+#ifdef FIM_DEBUG
+		std::cerr << "FileIndexMonitor::updateCycle() saved with hash:" << calchash;
+		std::cerr <<  std::endl;
+#endif
+
+		/* should clean up the previous cache.... */
+
+		/* flag as new info */
+		CacheData data;
+		data.pid = fi.root->id;
+		data.cid.type  = getCacheType();
+		data.cid.subid = 0;
+		data.path = path;
+		data.name = tmpname;
+		data.hash = calchash;
+		data.size = size;
+		data.recvd = time(NULL);
+
+		updateCache(data);
+
+#ifdef FIM_DEBUG
+		std::cerr << "FileIndexMonitor::updateCycle() called updateCache()";
+		std::cerr <<  std::endl;
+#endif
+
+		} fiMutex.unlock(); /* UNLOCKED DIRS */
+	}
+}
+
+	/* interface */
+void    FileIndexMonitor::setSharedDirectories(std::list<std::string> dirs)
+{
+	fiMutex.lock(); { /* LOCKED DIRS */
+
+	pendingDirs = true;
+	pendingDirList = dirs;
+
+	} fiMutex.unlock(); /* UNLOCKED DIRS */
+}
+
+bool    FileIndexMonitor::internal_setSharedDirectories()
+{
+	int i;
+	fiMutex.lock(); /* LOCKED DIRS */
+
+	if (!pendingDirs)
+	{
+		fiMutex.unlock(); /* UNLOCKED DIRS */
+		return false;
+	}
+		
+	pendingDirs = false;
+	pendingForceCacheWrite = true;
+	
+	/* clear old directories */
+	directoryMap.clear();
+	
+	/* iterate through the directories */
+	std::list<std::string>::iterator it;
+	std::map<std::string, std::string>::const_iterator cit;
+	for(it = pendingDirList.begin(); it != pendingDirList.end(); it++)
+	{
+		/* get the head directory */
+		std::string root_dir = *it;
+		std::string top_dir  = RsDirUtil::getTopDir(root_dir);
+	
+		/* if unique -> add, else add modifier  */
+		bool unique = false;
+		for(i = 0; !unique; i++)
+		{
+			std::string tst_dir = top_dir;
+			if (i > 0)
+			{
+				std::ostringstream out;
+				out << "-" << i;
+				tst_dir += out.str();
+			}
+			if (directoryMap.end()== (cit=directoryMap.find(tst_dir)))
+			{
+				unique = true;
+				/* add it! */
+				directoryMap[tst_dir.c_str()] = root_dir;
+				std::cerr << "Added [" << tst_dir << "] => " << root_dir << std::endl;
+			}
+		}
+	}
+
+	/* now we've decided on the 'root' dirs set them to the
+	 * fileIndex
+	 */
+	std::list<std::string> topdirs;
+	for(cit = directoryMap.begin(); cit != directoryMap.end(); cit++)
+	{
+		topdirs.push_back(cit->first);
+	}
+
+	fi.setRootDirectories(topdirs, 0);
+	
+	fiMutex.unlock(); /* UNLOCKED DIRS */
+
+	return true;
+}
+	
+	
+	
+
+/* lookup directory function */
+std::string FileIndexMonitor::findRealRoot(std::string rootdir)
+{
+	/**** MUST ALREADY BE LOCKED ****/ 
+	std::string realroot = "";
+
+	std::map<std::string, std::string>::const_iterator cit;
+	if (directoryMap.end()== (cit=directoryMap.find(rootdir)))
+	{
+		std::cerr << "FileIndexMonitor::findRealRoot() Invalid RootDir: ";
+		std::cerr << rootdir << std::endl;
+	}
+	else
+	{
+		realroot = cit->second;
+	}
+
+	return realroot;
+}
+
+
+
+bool FileIndexMonitor::hashFile(std::string fullpath, FileEntry &fent)
+{
+	std::string f_hash = fullpath + "/" + fent.name;
+	FILE *fd;
+	int  len;
+	SHA_CTX *sha_ctx = new SHA_CTX;
+	unsigned char sha_buf[SHA_DIGEST_LENGTH];
+	unsigned char gblBuf[512];
+
+	std::cerr << "File to hash = " << f_hash << std::endl;
+	if (NULL == (fd = fopen(f_hash.c_str(), "rb")))	return false;
+
+	SHA1_Init(sha_ctx);
+	while((len = fread(gblBuf,1, 512, fd)) > 0)
+	{
+		SHA1_Update(sha_ctx, gblBuf, len);
+	}
+
+	/* reading failed for some reason */
+	if (ferror(fd)) 
+	{
+		delete sha_ctx;
+		fclose(fd);
+		return false;
+	}
+
+	SHA1_Final(&sha_buf[0], sha_ctx);
+
+	/* TODO: Actually we should store the hash data as binary ... 
+	 * but then it shouldn't be put in a string.
+	 */
+
+        std::ostringstream tmpout;
+	for(int i = 0; i < SHA_DIGEST_LENGTH; i++)
+	{
+		tmpout << std::setw(2) << std::setfill('0') << std::hex << (unsigned int) (sha_buf[i]);
+	}
+	fent.hash = tmpout.str();
+
+	delete sha_ctx;
+	fclose(fd);
+	return true;
+}
+

libretroshare/src/dbase/fimonitor.h

@@ -0,0 +1,120 @@
+/*
+ * RetroShare FileCache Module: fimonitor.h
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef FILE_INDEX_MONITOR_H
+#define FILE_INDEX_MONITOR_H
+
+#include "dbase/findex.h"
+#include "dbase/cachestrapper.h"
+#include "util/rsthreads.h"
+
+/******************************************************************************************
+ * The Local Monitoring Class: FileIndexMonitor.
+ *
+ * This periodically scans the directory tree, and updates any modified directories/files.
+ *
+ *****************************************************************************************/
+
+/******************************************************************************************
+  STILL TODO:
+
+  (1) Implement Hash function.
+
+bool 	FileIndexMonitor::hashFile(std::string path, FileEntry &fi);
+
+  (2) Add Shared directory controls to Monitor.
+
+int   FileIndexMonitor::addSharedDirectory(std::path);
+int   FileIndexMonitor::removeSharedDirectory(std::path);
+std::string FileIndexMonitor::findRealRoot(std::string base);
+
+	These must be split into <base>/<top> and the mapping saved.
+	eg:  addSharedDirectory("c:/home/stuff/dir1") --> "c:/home/stuff" <-> "dir1"
+ 	This code has been written already, and can just be moved over.
+
+  FOR LATER:
+  (2) Port File/Directory lookup code to windoze. (or compile dirent.c under windoze)
+  (3) Add Load/Store interface to FileIndexMonitor. (later)
+  (4) Integrate with real Thread/Mutex code (last thing to do)
+
+******************************************************************************************/
+
+
+
+/******************************************************************************************
+ * FileIndexMonitor
+ *****************************************************************************************/
+
+class FileIndexMonitor: public CacheSource, public RsThread
+{
+	public:
+	FileIndexMonitor(std::string cachedir, std::string pid);
+virtual ~FileIndexMonitor();
+
+	/* external interface for filetransfer */
+bool    findLocalFile(std::string hash, std::string &fullpath, uint32_t &size);
+
+
+	/* Interacting with CacheSource */
+	/* overloaded from CacheSource */
+virtual bool loadCache(const CacheData &data);  /* called with stored data */
+bool 	updateCache(const CacheData &data);     /* we call when we have a new cache for others */
+
+
+	/* the FileIndexMonitor inner workings */
+virtual void 	run(); /* overloaded from RsThread */
+void 	updateCycle();
+
+void    setSharedDirectories(std::list<std::string> dirs);
+void    setPeriod(int insecs);
+
+	/* util fns */
+
+	private:
+
+	/* the mutex should be locked before calling... these. */
+std::string findRealRoot(std::string base);        /* To Implement */
+bool 	hashFile(std::string path, FileEntry &fi); /* To Implement */
+
+	/* data */
+
+	RsMutex fiMutex;
+
+	FileIndex fi;
+
+	int updatePeriod;
+	std::map<std::string, std::string> directoryMap; /* used by findRealRoot */
+
+	/* flags to kick - if we were busy or sleeping */
+	bool pendingDirs;
+	bool pendingForceCacheWrite;
+
+	std::list<std::string> pendingDirList;
+bool    internal_setSharedDirectories();
+
+};
+
+
+#endif
+
+

libretroshare/src/dbase/fimontest.cc

@@ -0,0 +1,87 @@
+/*
+ * RetroShare FileCache Module: fimontest.cc
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "findex.h"
+#include "fimonitor.h"
+
+#include <iostream>
+
+void usage(char *name)
+{
+	std::cerr << "Usage: " << name << " [-p <period> ] shareDir1 [shareDir2 [shareDir3 [...]]]";
+	std::cerr << std::endl;
+	exit(1);
+}
+
+
+
+int main(int argc, char **argv)
+{
+	/* handle commandline arguments */
+        int c;
+	int period = 60; /* recheck period in seconds */
+
+	while((c = getopt(argc, argv,"p:")) != -1)
+	{
+		switch(c)
+		{
+			case 'p':
+				period = atoi(optarg);
+				break;
+			default:
+				std::cerr << "Bad Option.";
+				std::cerr << std::endl;
+				usage(argv[0]);
+				break;
+		}
+	}
+
+	std::list<std::string> rootdirs;
+
+	/* add all the rest of the commandline arguments to rootdirs list */
+	for(; optind < argc; optind++)
+	{
+		rootdirs.push_back(argv[optind]);
+		std::cerr << "Adding shared directory: " << argv[optind] << std::endl;
+	}
+
+	if (rootdirs.size() < 1)
+	{
+		usage(argv[0]);
+	}
+
+	sleep(1);
+
+	FileIndexMonitor mon("", "OWN ID");
+
+	/* setup monitor */
+	mon.setPeriod(period);
+	mon.setSharedDirectories(rootdirs);
+
+	/* simulate running the thread */
+	mon.run();
+
+	return 1;
+}
+
+

libretroshare/src/dbase/findex.h

@@ -0,0 +1,229 @@
+/*
+ * RetroShare FileCache Module: findex.h
+ *
+ * Copyright 2004-2007 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef FILE_INDEX_H
+#define FILE_INDEX_H
+
+#include <string>
+#include <map>
+#include <list>
+#include <vector>
+
+class ostream;
+
+/******************************************************************************************
+ * The Key Data Types for the File Index:
+
+  FileEntry    : Information about a single file.
+  DirEntry     : Information about a directory and its children
+  PersonEntry  : Information the root of a FileIndex.
+
+  FileIndex : A Collection of root directories, with a set of functions to manipulate them. 
+	In terms of retroshare, There will be a single 'Local' FileIndex used to store
+	the shared files, and a set of 'Remote' FileIndices which are used to store
+	the available files of all the peers.
+
+******************************************************************************************/
+/******************************************************************************************
+  STILL TODO:
+
+  (1) Load/Store a FileIndex to file...
+	int FileIndex::loadIndex(FILE *input);
+	int FileIndex::saveIndex(FILE *input);
+
+      This can be done in a recursive manner, or handled completely within FileIndex.
+
+  (2) Search Functions for Partial File Names and Hashes. 
+
+	int FileIndex::searchHash(std::string hash, std::list<FileEntry> &results);
+	int FileIndex::searchTerms(std::list<string> terms, std::list<FileEntry> &results);
+
+      This is probably best done in a recursive manner.
+
+      The search could also be extended to handle complex Boolean searches such as : 
+      match (size > 100K) && (name contains 'Blue') .... if anyone is interested.
+      But this can get quite complicated, and can be left to a later date.
+
+******************************************************************************************/
+
+
+/******************************************************************************************
+ * FileEntry
+ *****************************************************************************************/
+
+class DirEntry;
+
+class FileEntry
+{
+	public:
+	FileEntry() :parent(NULL), row(-1) { return; }
+virtual ~FileEntry() { return; }
+
+virtual int print(std::ostream &out);
+
+	/* Data */
+	std::string name; 
+	std::string hash;
+	int size;         /* file size */
+	int modtime;      /* modification time - most recent mod time for a sub entry for dirs */
+	int pop;	  /* popularity rating */
+
+	int updtime;      /* last updated */
+
+	/* References for easy manipulation */
+	DirEntry *parent;
+	int       row;    
+};
+
+/******************************************************************************************
+ * DirEntry
+ *****************************************************************************************/
+
+class DirEntry: public FileEntry
+{
+	public:
+
+		/* cleanup */
+virtual		~DirEntry();
+
+		/* update local entries */
+DirEntry * 	updateDir(FileEntry fe, time_t updtime);
+FileEntry * 	updateFile(FileEntry fe, time_t updtime);
+
+
+int  		checkParentPointers();
+int 		updateChildRows();
+
+		/* remove local entries */
+int  		removeFile(std::string name);
+int  		removeDir(std::string name);
+int     	removeOldDir(std::string name, time_t old); /* checks ts first */
+
+		/* recursive cleanup */
+int  		removeOldEntries(time_t old, bool recursive); 
+
+		/* recursive searches */
+DirEntry *	findOldDirectory(time_t old);  
+DirEntry *	findDirectory(std::string path); 
+
+		/* recursive update directory mod/pop values */
+int	 	updateDirectories(std::string path, int pop, int modtime);
+
+		/* output */
+int 		print(std::ostream &out);
+
+int 		saveEntry(std::ostringstream &out);
+
+	/* Data */
+	std::string path;    /* full path (includes name) */
+	std::map<std::string, DirEntry *>  subdirs;
+	std::map<std::string, FileEntry *> files;
+
+	/* Inherited members from FileEntry:
+	int size 	  - count for dirs 
+	std::string name; - directory name 
+	std::string hash; - not used 
+	int size;         - not used 
+	int modtime;      - most recent modication time of any child file (recursive) 
+	int pop;	  - most popular child file (recursive)  
+	int updtime;      - last updated 
+	*/
+
+};
+
+/******************************************************************************************
+ * PersonEntry
+ *****************************************************************************************/
+
+class PersonEntry: public DirEntry
+{
+	public:
+	/* cleanup */
+	PersonEntry(std::string pid) : id(pid) { return; }
+virtual	~PersonEntry() { return; }
+
+DirEntry &operator=(DirEntry &src)
+{
+	DirEntry *pdest = this;
+	(*pdest) = src;
+	return src;
+}
+
+	/* Data */
+	std::string id;
+
+	/* Inherited members from FileEntry:
+	int size 	  - count for dirs 
+	std::string name; - directory name 
+	std::string hash; - not used 
+	int size;         - not used 
+	int modtime;      - most recent modication time of any child file (recursive) 
+	int pop;	  - most popular child file (recursive)  
+	int updtime;      - last updated 
+	*/
+
+};
+
+/******************************************************************************************
+ * FileIndex
+ *****************************************************************************************/
+
+class Expression;
+
+class FileIndex
+{
+	public:
+	FileIndex(std::string pid);
+	~FileIndex();
+
+	/* control root entries */
+int	setRootDirectories(std::list<std::string> inlist, time_t utime);
+int	getRootDirectories(std::list<std::string> &outlist);
+
+	/* update (index building) */
+DirEntry  * updateDirEntry(std::string path, FileEntry fe, time_t utime);
+FileEntry * updateFileEntry(std::string path, FileEntry fe, time_t utime);
+
+DirEntry  * findOldDirectory(time_t old); /* finds directories older than old */
+int     removeOldDirectory(std::string fpath, std::string name, time_t old);
+
+int	cleanOldEntries(time_t old);  /* removes entries older than old */
+
+	/* debug */
+int	printFileIndex(std::ostream &out);
+
+	/* load/save to file */
+int 	loadIndex(std::string filename, std::string expectedHash, uint32_t size);
+int 	saveIndex(std::string filename, std::string &fileHash, uint32_t &size);
+
+	/* search through this index */
+int 	searchTerms(std::list<std::string> terms, std::list<FileEntry *> &results);
+int 	searchHash(std::string hash, std::list<FileEntry *> &results);
+int     searchBoolExp(Expression * exp, std::list<FileEntry *> &results);
+
+	PersonEntry *root;
+};
+
+
+#endif
+

libretroshare/src/dbase/fisavetest.cc

@@ -0,0 +1,118 @@
+/*
+ * RetroShare FileCache Module: fisavetest.cc
+ *     
+ * Copyright 2004-2007 by Kefei Zhou.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "dbase/findex.h"
+#include <iostream>
+
+FileIndex *createBasicFileIndex(time_t age);
+
+int main()
+{
+	FileIndex *fi1 = createBasicFileIndex(100);
+	FileIndex *fi2 = new FileIndex("A SILLY ID");
+
+	fi1->printFileIndex(std::cout);
+	std::string fhash;
+	uint32_t size;
+	fi1->saveIndex("test.index", fhash, size);
+
+	std::cout << " Saved Index: Size: " << size << " Hash: " << fhash << std::endl;
+	std::cout << " -- new file index -- " << std::endl;
+	
+	fi2->loadIndex("test.index", fhash, size);
+	fi2->printFileIndex(std::cout);
+
+	delete fi1;
+	delete fi2;
+
+	return 1;
+}
+
+
+FileIndex *createBasicFileIndex(time_t age)
+{
+	FileIndex *fi = new FileIndex("A SILLY ID");
+
+	FileEntry fe;
+
+	std::list<std::string> rootdirs;
+	rootdirs.push_back("base1");
+	rootdirs.push_back("base2");
+	rootdirs.push_back("base3");
+
+	fi -> setRootDirectories(rootdirs, age);
+
+	/* add some entries */
+	fe.name = "dir1";
+	fi -> updateDirEntry("base1",fe, age);
+	fe.name = "dir2";
+	fi -> updateDirEntry("base1",fe, age);
+
+	fe.name = "dir01";
+	fi -> updateDirEntry("/base1/dir1/",fe, age);
+
+	fe.name = "dir001";
+	fi -> updateDirEntry("/base1/dir1/dir01/",fe, age);
+
+	fe.name = "file1";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+	fe.name = "file2";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+	fe.name = "file3";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+	fe.name = "file4";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+
+
+	fe.name = "dir2";
+	fi -> updateDirEntry("/base1",fe, age);
+	fe.name = "file5";
+	fi -> updateFileEntry("/base1/dir2/",fe, age);
+	fe.name = "file6";
+	fi -> updateFileEntry("/base1/dir2/",fe, age);
+	fe.name = "file7";
+	fi -> updateFileEntry("/base1/dir2/",fe, age);
+	fe.name = "file8";
+	fi -> updateFileEntry("/base1/",fe, age);
+
+
+	fe.name = "dir3";
+	fi -> updateDirEntry("/base1/dir2/",fe, age);
+	fe.name = "file10";
+	fi -> updateFileEntry("/base1/dir2/dir3",fe, age);
+	fe.name = "file11";
+	fi -> updateFileEntry("/base1/dir2/dir3",fe, age);
+	fe.name = "file12";
+	fi -> updateFileEntry("/base1/dir2/dir3",fe, age);
+
+
+	fe.name = "dir4";
+	fi -> updateDirEntry("/base3/",fe, age);
+	fe.name = "file20";
+	fi -> updateFileEntry("/base3/dir4/",fe, age);
+	fe.name = "file21";
+	fi -> updateFileEntry("/base3/dir4",fe, age);
+
+	return fi;
+}
+

libretroshare/src/dbase/fistore.cc

@@ -0,0 +1,521 @@
+/*
+ * RetroShare FileCache Module: fistore.cc
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "dbase/fistore.h"
+#include "rsiface/rsexpr.h"
+
+FileIndexStore::FileIndexStore(CacheTransfer *cft, 
+		NotifyBase *cb_in, RsPeerId ownid, std::string cachedir)
+	:CacheStore(CACHE_TYPE_FILE_INDEX, false, cft, cachedir), 
+		localId(ownid), localindex(NULL), cb(cb_in)
+{ 
+	return;
+}
+
+FileIndexStore::~FileIndexStore() 
+{
+	/* clean up the Index */
+	return;
+}
+
+/***
+ * #define FIS_DEBUG2 1
+ * #define FIS_DEBUG 1
+ **/
+
+	  /* actual load, once data available */
+int FileIndexStore::loadCache(const CacheData &data)
+{
+
+#ifdef FIS_DEBUG2
+	std::cerr << "FileIndexStore::loadCache() hash: " << data.hash << std::endl;
+	std::cerr << "FileIndexStore::loadCache() path: " << data.path << std::endl;
+	std::cerr << "FileIndexStore::loadCache() name: " << data.name << std::endl;
+	std::cerr << "FileIndexStore::loadCache() size: " << data.size << std::endl;
+#endif
+
+	/* do Callback */
+	AboutToModify();
+
+	/* lock it up */
+	lockData();
+
+	FileIndex *fiold = NULL;
+	bool local = (data.pid == localId);
+
+	std::map<RsPeerId, FileIndex *>::iterator it;
+	/* remove old cache */
+	if (local)
+	{
+		fiold = localindex;
+		localindex = NULL;
+	}
+	else if (indices.end() != (it = indices.find(data.pid)))
+	{
+		fiold = it->second;
+		indices.erase(it);
+		//delete fi;
+	}
+
+	/* load Cache */
+	FileIndex *finew = new FileIndex(data.pid);
+
+	if (finew->loadIndex(data.path + '/' + data.name, data.hash, data.size))
+	{
+#ifdef FIS_DEBUG2
+		std::cerr << "FileIndexStore::loadCache() Succeeded!" << std::endl;
+#endif
+		/* set the name */
+		finew->root->name = data.pname;
+		if (local)
+		{
+			localindex = finew;
+		}
+		else
+		{
+			indices[data.pid] = finew;
+		}
+		delete fiold;
+
+		/* store in tale */
+		locked_storeCacheEntry(data);
+	}
+	else
+	{
+#ifdef FIS_DEBUG2
+		std::cerr << "FileIndexStore::loadCache() Failed!" << std::endl;
+#endif
+		/* reinstall the old one! */
+		delete finew;
+		if (fiold)
+		{
+			if (local)
+			{
+				localindex = fiold;
+			}
+			else
+			{
+				indices[data.pid] = fiold;
+			}
+		}
+	}
+
+	/* need to correct indices(row) for the roots of the FileIndex */
+	int i = 0;
+	for(it = indices.begin(); it != indices.end(); it++)
+	{
+		(it->second)->root->row = i++;
+	}
+	if (localindex)
+	{
+		localindex->root->row = 0;
+	}
+
+	unlockData();
+
+	ModCompleted();
+	bool ret = false;
+	return ret;
+}
+
+	/* Search Interface - For Directory Access */
+int FileIndexStore::RequestDirDetails(std::string uid, std::string path, DirDetails &details)
+{
+	/* lock it up */
+	lockData();
+
+	std::map<RsPeerId, FileIndex *>::iterator it;
+	it = indices.find(uid);
+	bool found = true;
+	if (it == indices.end())
+	{
+		//DirEntry *fdir = (it->second).lookupDirectory(path);
+		/* translate it 
+		 */
+		found = false;
+	}
+	else
+	{
+		found = false;
+	}
+
+	unlockData();
+	return found;
+}
+
+int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, uint32_t flags)
+{
+	lockData();
+	bool found = true;
+	std::map<RsPeerId, FileIndex *>::iterator pit;
+
+	/* so cast *ref to a DirEntry */
+	FileEntry *file = (FileEntry *) ref;
+	DirEntry *dir = dynamic_cast<DirEntry *>(file);
+	PersonEntry *person;
+	/* root case */
+
+#ifdef FIS_DEBUG
+	std::cerr << "FileIndexStore::RequestDirDetails() CHKS" << std::endl;
+	for(pit = indices.begin(); pit != indices.end(); pit++)
+	{
+		(pit->second)->root->checkParentPointers();
+	}
+#endif
+
+	if (!ref)
+	{
+#ifdef FIS_DEBUG
+		std::cerr << "FileIndexStore::RequestDirDetails() ref=NULL (root)" << std::endl;
+#endif
+		if (flags & DIR_FLAGS_LOCAL)
+		{
+			/* local only */
+			if (localindex)
+			{
+				DirStub stub;
+				stub.type = DIR_TYPE_PERSON;
+				stub.name = localindex->root->name;
+				stub.ref  = localindex->root;
+				details.children.push_back(stub);
+				details.count = 1;
+			}
+			else
+			{
+				details.count = 0;
+			}
+
+			details.parent = NULL;
+			details.prow = 0;
+			details.ref = NULL;
+			details.type = DIR_TYPE_ROOT;
+			details.name = "";
+			details.hash = "";
+			details.path = "";
+			details.age = 0;
+			details.rank = 0;
+		}
+		else
+		{
+			/* get remote root entries */
+			for(pit = indices.begin(); pit != indices.end(); pit++)
+			{
+				/* 
+				 */
+				DirStub stub;
+				stub.type = DIR_TYPE_PERSON;
+				stub.name = (pit->second)->root->name;
+				stub.ref =  (pit->second)->root;
+	
+				details.children.push_back(stub);
+			}
+			details.parent = NULL;
+			details.prow = 0;
+			details.ref = NULL;
+			details.type = DIR_TYPE_ROOT;
+			details.name = "";
+			details.hash = "";
+			details.path = "";
+			details.count = indices.size();
+			details.age = 0;
+			details.rank = 0;
+		}
+	}
+	else 
+	{
+		if (dir) /* has children --- fill */
+		{
+#ifdef FIS_DEBUG
+			std::cerr << "FileIndexStore::RequestDirDetails() ref=dir" << std::endl;
+#endif
+			std::map<std::string, FileEntry *>::iterator fit;
+			std::map<std::string, DirEntry *>::iterator dit;
+			/* extract all the entries */
+			for(dit = dir->subdirs.begin(); 
+				dit != dir->subdirs.end(); dit++)
+			{
+				DirStub stub;
+				stub.type = DIR_TYPE_DIR;
+				stub.name = (dit->second) -> name;
+				stub.ref  = (dit->second);
+	
+				details.children.push_back(stub);
+			}
+	
+			for(fit = dir->files.begin(); 
+				fit != dir->files.end(); fit++)
+			{
+				DirStub stub;
+				stub.type = DIR_TYPE_FILE;
+				stub.name = (fit->second) -> name;
+				stub.ref  = (fit->second);
+	
+				details.children.push_back(stub);
+			}
+
+			details.type = DIR_TYPE_DIR;
+			details.hash = "";
+			details.count = dir->subdirs.size() + 
+					dir->files.size();
+		}
+		else
+		{
+#ifdef FIS_DEBUG
+			std::cerr << "FileIndexStore::RequestDirDetails() ref=file" << std::endl;
+#endif
+			details.type = DIR_TYPE_FILE;
+			details.count = file->size;
+		}
+	
+#ifdef FIS_DEBUG
+		std::cerr << "FileIndexStore::RequestDirDetails() name: " << file->name << std::endl;
+#endif
+		details.ref = file;
+		details.name = file->name;
+		details.hash = file->hash;
+		details.age = time(NULL) - file->modtime;
+		details.rank = file->pop;
+
+		/* TODO Path */
+		details.path = "";
+
+		/* find parent pointer, and row */
+		DirEntry *parent = file->parent;
+		if (!parent) /* then must be root */
+		{
+			details.parent = NULL;
+			details.prow = 0;
+		}
+		else
+		{
+			details.parent = parent;
+			details.prow = parent->row;
+		}
+
+		/* find peer id */
+		parent = dir;
+		if (!dir)
+		{
+			/* cannot be null -> no files at root level */
+			parent=file->parent;
+		}
+
+		while(parent->parent)
+			parent = parent->parent;
+		
+		/* we should end up on the PersonEntry */
+		if (NULL == (person = dynamic_cast<PersonEntry *>(parent)))
+		{
+			std::cerr << "Major Error- Not PersonEntry!";
+			exit(1);
+		}
+
+		details.id = person->id;
+	}
+
+	unlockData();
+	return found;
+}
+
+
+int FileIndexStore::SearchHash(std::string hash, std::list<FileDetail> &results)
+{
+	lockData();
+	std::map<RsPeerId, FileIndex *>::iterator pit;
+	std::list<FileEntry *>::iterator rit; 
+	std::list<FileEntry *> firesults;
+
+	time_t now = time(NULL);
+
+#ifdef FIS_DEBUG
+	std::cerr << "FileIndexStore::SearchHash()" << std::endl;
+#endif
+	for(pit = indices.begin(); pit != indices.end(); pit++)
+	{
+		firesults.clear();
+
+		(pit->second)->searchHash(hash, firesults);
+		/* translate results */
+		for(rit = firesults.begin(); rit != firesults.end(); rit++)
+		{
+			FileDetail fd;
+			fd.id = pit->first;
+			fd.name = (*rit)->name;
+			fd.hash = (*rit)->hash;
+			fd.path = ""; /* TODO */
+			fd.size = (*rit)->size;
+			fd.age  = now - (*rit)->modtime;
+			fd.rank = (*rit)->pop;
+
+			results.push_back(fd);
+		}
+
+	}
+
+	unlockData();
+	return results.size();
+}
+
+
+int FileIndexStore::SearchKeywords(std::list<std::string> keywords, std::list<FileDetail> &results)
+{
+	lockData();
+	std::map<RsPeerId, FileIndex *>::iterator pit;
+	std::list<FileEntry *>::iterator rit; 
+	std::list<FileEntry *> firesults;
+
+	time_t now = time(NULL);
+
+#ifdef FIS_DEBUG
+	std::cerr << "FileIndexStore::SearchKeywords()" << std::endl;
+#endif
+	for(pit = indices.begin(); pit != indices.end(); pit++)
+	{
+		firesults.clear();
+
+		(pit->second)->searchTerms(keywords, firesults);
+		/* translate results */
+		for(rit = firesults.begin(); rit != firesults.end(); rit++)
+		{
+			FileDetail fd;
+			fd.id = pit->first;
+			fd.name = (*rit)->name;
+			fd.hash = (*rit)->hash;
+			fd.path = ""; /* TODO */
+			fd.size = (*rit)->size;
+			fd.age  = now - (*rit)->modtime;
+			fd.rank = (*rit)->pop;
+
+			results.push_back(fd);
+		}
+
+	}
+	if (localindex)
+	{
+		firesults.clear();
+
+		localindex->searchTerms(keywords, firesults);
+		/* translate results */
+		for(rit = firesults.begin(); rit != firesults.end(); rit++)
+		{
+			FileDetail fd;
+			fd.id = "Local"; //localId;
+			fd.name = (*rit)->name;
+			fd.hash = (*rit)->hash;
+			fd.path = ""; /* TODO */
+			fd.size = (*rit)->size;
+			fd.age  = now - (*rit)->modtime;
+			fd.rank = (*rit)->pop;
+
+			results.push_back(fd);
+		}
+
+	}
+
+	unlockData();
+	return results.size();
+}
+
+
+int FileIndexStore::searchBoolExp(Expression * exp, std::list<FileDetail> &results)
+{
+	lockData();
+	std::map<RsPeerId, FileIndex *>::iterator pit;
+	std::list<FileEntry *>::iterator rit; 
+	std::list<FileEntry *> firesults;
+
+	time_t now = time(NULL);
+
+#ifdef FIS_DEBUG
+	std::cerr << "FileIndexStore::searchBoolExp()" << std::endl;
+#endif
+	for(pit = indices.begin(); pit != indices.end(); pit++)
+	{
+		firesults.clear();
+
+		(pit->second)->searchBoolExp(exp, firesults);
+
+		/* translate results */
+		for(rit = firesults.begin(); rit != firesults.end(); rit++)
+		{
+			FileDetail fd;
+			fd.id = pit->first;
+			fd.name = (*rit)->name;
+			fd.hash = (*rit)->hash;
+			fd.path = ""; /* TODO */
+			fd.size = (*rit)->size;
+			fd.age  = now - (*rit)->modtime;
+			fd.rank  = (*rit)->pop;
+
+			results.push_back(fd);
+		}
+
+	}
+
+	/* finally search local files */
+	if (localindex)
+	{
+		firesults.clear();
+
+		localindex->searchBoolExp(exp, firesults);
+
+		/* translate results */
+		for(rit = firesults.begin(); rit != firesults.end(); rit++)
+		{
+			FileDetail fd;
+			fd.id = "Local"; //localId;
+			fd.name = (*rit)->name;
+			fd.hash = (*rit)->hash;
+			fd.path = ""; /* TODO */
+			fd.size = (*rit)->size;
+			fd.age  = now - (*rit)->modtime;
+			fd.rank  = (*rit)->pop;
+
+			results.push_back(fd);
+		}
+
+	}
+
+
+	unlockData();
+	return results.size();
+}
+
+int FileIndexStore::AboutToModify()
+{
+	if (cb)
+		cb->notifyListPreChange(NOTIFY_LIST_DIRLIST, 0);
+
+	return 1;
+}
+
+
+int FileIndexStore::ModCompleted()
+{
+	if (cb)
+		cb->notifyListChange(NOTIFY_LIST_DIRLIST, 0);
+
+	return 1;
+}
+
+

libretroshare/src/dbase/fistore.h

@@ -0,0 +1,100 @@
+/*
+ * RetroShare FileCache Module: fistore.h
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#ifndef MRK_FILE_INDEX_STORE_H
+#define MRK_FILE_INDEX_STORE_H
+
+
+/**********
+ * Stores the FileCaches of the Peers
+ * must implement 'loadCache' to 
+ *
+ * This class is also accessed by the GUI....
+ * and the FileTransfer class.
+ *
+ */
+
+#include "dbase/findex.h"
+#include "dbase/cachestrapper.h"
+#include "rsiface/rsiface.h"
+
+class FileStoreResult
+{
+	public:
+	std::string id;
+	std::string path;
+	std::string hash;
+	std::string name;
+};
+
+class NotifyCallback
+{
+        public:
+        NotifyCallback() 	{ return; }
+virtual ~NotifyCallback() 	{ return; }
+virtual void AboutToModify()	{ return; }
+virtual void ModCompleted()     { return; }
+};
+
+
+class Expression;
+
+class FileIndexStore: public CacheStore
+{
+	public:
+
+	FileIndexStore(CacheTransfer *cft, NotifyBase *cb_in, 
+			RsPeerId ownid, std::string cachedir);
+virtual ~FileIndexStore();
+
+	/* virtual functions overloaded by cache implementor */
+virtual int loadCache(const CacheData &data);	  /* actual load, once data available */
+
+	/* Search Interface - For FileTransfer Lookup */
+	int SearchHash(std::string hash, std::list<FileDetail> &results);
+
+	/* Search Interface - For Search Interface */
+	int SearchKeywords(std::list<std::string> terms, std::list<FileDetail> &results);
+
+	/* Search Interface - for Adv Search Interface */
+	int searchBoolExp(Expression * exp, std::list<FileDetail> &results);
+
+
+	/* Search Interface - For Directory Access */
+	int RequestDirDetails(std::string uid, std::string path, DirDetails &details);
+	int RequestDirDetails(void *ref, DirDetails &details, uint32_t flags);
+
+	private:
+	int AboutToModify();
+	int ModCompleted();
+
+	std::map<RsPeerId, FileIndex *> indices;
+
+	RsPeerId   localId;
+	FileIndex *localindex;
+
+	NotifyBase *cb;
+};
+
+
+#endif

libretroshare/src/dbase/fitest2.cc

@@ -0,0 +1,213 @@
+/*
+ * RetroShare FileCache Module: fitest2.cc
+ *   
+ * Copyright 2004-2007 by Robert Fernie.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "dbase/findex.h"
+
+#include <iostream>
+
+FileIndex *createBasicFileIndex(time_t age);
+
+int test1(FileIndex *fi);
+int test2(FileIndex *fi);
+
+int main()
+{
+	FileIndex *fi = createBasicFileIndex(100);
+
+	test1(fi);
+
+	delete fi;
+
+	return 1;
+}
+
+int test1(FileIndex *fi)
+{
+	/* in this test we are going to get the old directories - and update them */
+	time_t stamp = 200;
+
+	DirEntry *olddir = NULL;
+	FileEntry fe;
+	while((olddir = fi -> findOldDirectory(stamp)))
+	{
+		/* update the directories and files here */
+		std::map<std::string, DirEntry *>::iterator  dit;
+		std::map<std::string, FileEntry *>::iterator fit;
+		
+		/* update this dir */
+		fe.name = olddir->name;
+		fi -> updateDirEntry(olddir->parent->path, fe, stamp);
+
+		/* update subdirs */
+		for(dit = olddir->subdirs.begin(); dit != olddir->subdirs.end(); dit++)
+		{
+			fe.name = (dit->second)->name;
+			/* set the age as out-of-date so that it gets checked */
+			fi -> updateDirEntry(olddir->path, fe, 0);
+		}
+
+		/* update files */
+		for(fit = olddir->files.begin(); fit != olddir->files.end(); fit++)
+		{
+			fe.name = (fit->second)->name;
+			fi -> updateFileEntry(olddir->path, fe, stamp);
+		}
+
+	  	/* clean up the dir (should have no effect) */
+	  	fi -> removeOldDirectory(olddir->parent->path, olddir->name, stamp);
+		fi -> printFileIndex(std::cout);
+	}
+
+	fi -> printFileIndex(std::cout);
+
+	return 1;
+}
+
+
+int test2(FileIndex *fi)
+{
+	/* in this test we are going to simulate that 2 directories have disappeared */
+	time_t stamp = 200;
+
+	DirEntry *olddir = NULL;
+	FileEntry fe;
+	bool missingdir = false;
+	int i = 0;
+
+	while((olddir = fi -> findOldDirectory(stamp)))
+	{
+	  missingdir = false;
+	  if (i % 2 == 0)
+	  {
+		std::cerr << " Simulating that dir doesnt exist :" << olddir->path;
+		std::cerr << std::endl;
+		missingdir = true;
+	  }
+	  i++;
+
+          if (!missingdir)
+          {
+		/* update the directories and files here */
+		std::map<std::string, DirEntry *>::iterator  dit;
+		std::map<std::string, FileEntry *>::iterator fit;
+		
+		/* update this dir */
+		fe.name = olddir->name;
+		fi -> updateDirEntry(olddir->parent->path, fe, stamp);
+
+		/* update subdirs */
+		for(dit = olddir->subdirs.begin(); dit != olddir->subdirs.end(); dit++)
+		{
+			fe.name = (dit->second)->name;
+			/* set the age as out-of-date so that it gets checked */
+			fi -> updateDirEntry(olddir->path, fe, 0);
+		}
+
+		/* update files */
+		for(fit = olddir->files.begin(); fit != olddir->files.end(); fit++)
+		{
+			fe.name = (fit->second)->name;
+			fi -> updateFileEntry(olddir->path, fe, stamp);
+		}
+	  }
+	  /* clean up the dir */
+	  fi -> removeOldDirectory(olddir->parent->path, olddir->name, stamp);
+
+	  fi -> printFileIndex(std::cout);
+	}
+
+	fi -> printFileIndex(std::cout);
+
+	return 1;
+}
+
+
+
+
+
+FileIndex *createBasicFileIndex(time_t age)
+{
+	FileIndex *fi = new FileIndex("A SILLY ID");
+
+	FileEntry fe;
+
+	/* print empty FileIndex */
+	fi -> printFileIndex(std::cout);
+
+	std::list<std::string> rootdirs;
+	rootdirs.push_back("base1");
+	rootdirs.push_back("base2");
+	rootdirs.push_back("base3");
+
+	fi -> setRootDirectories(rootdirs, age);
+
+	/* add some entries */
+	fe.name = "dir1";
+	fi -> updateDirEntry("base1",fe, age);
+	fe.name = "file1";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+	fe.name = "file2";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+	fe.name = "file3";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+	fe.name = "file4";
+	fi -> updateFileEntry("/base1/dir1/",fe, age);
+
+	fe.name = "dir2";
+	fi -> updateDirEntry("/base1",fe, age);
+	fe.name = "file5";
+	fi -> updateFileEntry("/base1/dir2/",fe, age);
+	fe.name = "file6";
+	fi -> updateFileEntry("/base1/dir2/",fe, age);
+	fe.name = "file7";
+	fi -> updateFileEntry("/base1/dir2/",fe, age);
+	fe.name = "file8";
+	fi -> updateFileEntry("/base1/",fe, age);
+
+
+	fe.name = "dir3";
+	fi -> updateDirEntry("/base1/dir2/",fe, age);
+	fe.name = "file10";
+	fi -> updateFileEntry("/base1/dir2/dir3",fe, age);
+	fe.name = "file11";
+	fi -> updateFileEntry("/base1/dir2/dir3",fe, age);
+	fe.name = "file12";
+	fi -> updateFileEntry("/base1/dir2/dir3",fe, age);
+
+
+	fe.name = "dir4";
+	fi -> updateDirEntry("/base3/",fe, age);
+	fe.name = "file20";
+	fi -> updateFileEntry("/base3/dir4/",fe, age);
+	fe.name = "file21";
+	fi -> updateFileEntry("/base3/dir4",fe, age);
+
+	// one that will fail.
+	fe.name = "file20";
+	fi -> updateFileEntry("/base3/",fe, age);
+
+	fi -> printFileIndex(std::cout);
+
+	return fi;
+}
+

libretroshare/src/dbase/ftest.cc

@@ -0,0 +1,111 @@
+/*
+ * "$Id: ftest.cc,v 1.5 2007-02-18 21:46:49 rmf24 Exp $"
+ *
+ * Other Bits for RetroShare.
+ *
+ * Copyright 2004-2006 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+/* Example Test of filedex index */
+
+#include "filedex.h"
+#include <iostream>
+
+int main()
+{
+	filedex fd;
+	std::string term;
+	std::list<DirItem> dirs;
+	std::list<std::string> terms;
+	std::list<fdex *> results;
+	std::list<fdex *>::iterator it;
+
+	dirs.push_back(DirItem("/usr/local", "", 1));
+	dirs.push_back(DirItem("/var/log", "", 2));
+
+	fd.load(dirs);
+
+	// now show the root directories...
+	results.clear();
+	results = fd.dirlisting("");
+
+	std::cerr << "Root Directory Results:" << std::endl;
+	for(it = results.begin(); it != results.end(); it++)
+	{
+		std::cerr << "Full Path: " << (*it) -> path << std::endl;
+		std::cerr << "\tFile:" << (*it) -> file << std::endl;
+		std::cerr << "\tExt:" << (*it) -> ext;
+		std::cerr << " Size: " << (*it) -> len << std::endl;
+		std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
+	}
+
+	while(1)
+	{
+		std::cerr << "Enter Search Term :";
+		std::cin >> term;
+		std::cerr << "Searching For:" << term << std::endl;
+		terms.clear();
+		terms.push_back(term);
+
+		results.clear();
+		results = fd.search(terms, 10, false);
+
+		std::cerr << "Search Results:" << std::endl;
+		for(it = results.begin(); it != results.end(); it++)
+		{
+			std::cerr << "Full Path: " << (*it) -> path << std::endl;
+			std::cerr << "\tFile:" << (*it) -> file << std::endl;
+			std::cerr << "\tExt:" << (*it) -> ext;
+			std::cerr << " Size: " << (*it) -> len << std::endl;
+			std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
+		}
+		
+		results.clear();
+		results = fd.dirlisting(term);
+
+		std::cerr << "FileName Results:" << std::endl;
+		for(it = results.begin(); it != results.end(); it++)
+		{
+			std::cerr << "Full Path: " << (*it) -> path << std::endl;
+			std::cerr << "\tFile:" << (*it) -> file << std::endl;
+			std::cerr << "\tExt:" << (*it) -> ext;
+			std::cerr << " Size: " << (*it) -> len << std::endl;
+			std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
+		}
+
+		results.clear();
+		results = fd.findfilename(term);
+
+		std::cerr << "FileName Results:" << std::endl;
+		for(it = results.begin(); it != results.end(); it++)
+		{
+			std::cerr << "Full Path: " << (*it) -> path << std::endl;
+			std::cerr << "\tFile:" << (*it) -> file << std::endl;
+			std::cerr << "\tExt:" << (*it) -> ext;
+			std::cerr << " Size: " << (*it) -> len << std::endl;
+			std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
+		}
+	}
+
+	//sleep(5);
+	return 1;
+}
+
+

libretroshare/src/dbase/looktest.cc

@@ -0,0 +1,89 @@
+/*
+ * "$Id: looktest.cc,v 1.1 2007-02-19 20:08:30 rmf24 Exp $"
+ *
+ * Other Bits for RetroShare.
+ *
+ * Copyright 2004-2006 by Robert Fernie.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+
+
+
+#include "filedex.h"
+#include "dbase/filelook.h"
+#include <iostream>
+
+#include "pqi/pqi.h"
+
+int main()
+{
+	fileLook *fl = new fileLook();
+	std::string term;
+	std::list<std::string> dirs;
+	std::list<std::string> terms;
+	std::list<fdex *> results;
+	std::list<fdex *>::iterator it;
+
+	dirs.push_back("/mnt/disc2/extra/rmf24/mp3s/good");
+	dirs.push_back("/mnt/disc2/extra/rmf24/mp3s/marks");
+	dirs.push_back("/mnt/disc2/extra/rmf24/mp3s/incoming");
+
+	fl -> start(); /* background look thread */
+
+	std::cerr << "FileLook Thread started" << std::endl;
+
+	fl -> setSharedDirectories(dirs);
+
+	PQFileItem *i = new PQFileItem();
+
+
+	std::cerr << "test Lookup ..." << std::endl;
+	fl -> lookUpDirectory(i);
+	std::cerr << "Entering Main Loop" << std::endl;
+
+	while(1)
+	{
+		std::cerr << "Enter Search Term :";
+		std::cin >> term;
+		std::cerr << "Searching For:" << term << std::endl;
+		terms.clear();
+		terms.push_back(term);
+
+		PQFileItem *i = new PQFileItem();
+		i -> path = term;
+
+		std::cerr << "Root Lookup ..." << std::endl;
+		fl -> lookUpDirectory(i);
+		std::cerr << "LookUp done" << std::endl;
+
+		while(NULL != (i = (PQFileItem *) fl -> getResult()))
+		{
+			std::cerr << "Results:" << std::endl;
+			i -> print(std::cerr);
+			std::cerr << std::endl;
+		}
+
+	}
+
+	//sleep(5);
+	return 1;
+}
+
+

libretroshare/src/dbase/rsexpr.cc

@@ -0,0 +1,149 @@
+/*
+ * rs-core/src/dbase: rsexpr.cc
+ *
+ * RetroShare C++ Interface.
+ *
+ * Copyright 2007-2008 by Kashif Kaleem.
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+
+#include "dbase/findex.h"
+#include "rsiface/rsexpr.h"
+#include <functional>
+
+
+/******************************************************************************************
+eval functions of relational expressions. 
+
+******************************************************************************************/
+
+bool DateExpression::eval(FileEntry *file)
+{
+		return evalRel(file->modtime);	
+}
+
+bool SizeExpression::eval(FileEntry *file)
+{
+	return evalRel(file->size);	
+}
+
+bool PopExpression::eval(FileEntry *file)
+{
+	return evalRel(file->pop);
+}
+
+/******************************************************************************************
+Code for evaluating string expressions
+
+******************************************************************************************/
+
+bool NameExpression::eval(FileEntry *file) 
+{
+	return evalStr(file->name);
+}
+
+bool PathExpression::eval(FileEntry *file){
+	std::string path;
+	/*Construct the path of this file*/
+	DirEntry * curr = file->parent;
+	while ( curr != NULL ){
+		path = curr->name+"/"+ path;
+		curr = curr->parent;
+	}
+	return evalStr(path);
+}
+
+bool ExtExpression::eval(FileEntry *file){
+	std::string ext;
+	/*Get the part of the string after the last instance of . in the filename */
+	unsigned int index = file->name.find_last_of('.');
+	if (index != std::string::npos) {
+		ext = file->name.substr(index+1);
+		if (ext != "" ){
+			return evalStr(ext);	
+		}
+	}
+	return false;
+}
+
+bool HashExpression::eval(FileEntry *file){
+	return evalStr(file->hash);
+}
+
+/*Check whether two strings are 'equal' to each other*/
+static bool StrEquals(const std::string & str1, const std::string & str2, 
+			   bool IgnoreCase ){
+	if ( str1.size() != str2.size() ){
+		return false;
+	} else if (IgnoreCase) {
+		std::equal( str1.begin(), str1.end(), 
+						   str2.begin(), CompareCharIC() );
+	}
+	return std::equal( str1.begin(), str1.end(), 
+						   str2.begin());
+}
+
+/*Check whether one string contains the other*/
+static bool StrContains( std::string & str1, std::string & str2, 
+				  bool IgnoreCase){
+
+	std::string::const_iterator iter ;
+	if (IgnoreCase) {
+		iter = std::search( str1.begin(), str1.end(),
+					   		str2.begin(), str2.end(), CompareCharIC() );		
+	} else {
+		iter = std::search( str1.begin(), str1.end(),
+					   		str2.begin(), str2.end());		
+	}
+	
+	return ( iter != str1.end() );
+}
+
+
+bool StringExpression :: evalStr ( std::string &str ){
+	std::list<std::string>::iterator iter;
+	switch (Op) {
+		case ContainsAllStrings:
+			for ( iter = terms.begin(); iter != terms.end(); iter++ ) {	
+				if ( StrContains (str, *iter, IgnoreCase) == false ){
+					return false;	
+				}
+			}
+			return true;
+		break;
+		case ContainsAnyStrings:
+			for ( iter = terms.begin(); iter != terms.end(); iter++ ) {
+				if ( StrContains (str,*iter, IgnoreCase) == true ) {
+					return true;	
+				}
+			}
+		break;
+		case EqualsString:
+			for ( iter = terms.begin(); iter != terms.end(); iter++ ) {
+				if ( StrEquals (str,*iter, IgnoreCase) == true ) {
+					return true;	
+				}
+			}
+		break;
+		default:
+			return false;
+	}
+	return false;
+}

libretroshare/src/dbase/searchtest.cc

@@ -0,0 +1,74 @@
+/*
+ * RetroShare FileCache Module: searchtest.cc
+ *   
+ * Copyright 2004-2007 by Kefei Zhou.
+ *     
+ * This library is free software; you can redistribute it and/or 
+ * modify it under the terms of the GNU Library General Public
+ * License Version 2 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful, 
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of 
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU  
+ * Library General Public License for more details. 
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "retroshare@lunamutt.com".
+ *
+ */
+
+#include "dbase/findex.h"
+#include <iostream>
+#include <fstream>
+
+int main()
+{
+
+	std::cout << std::string::npos << std::endl;
+	std::string testfile = "searchtest.index";
+	std::string fhash = "6851c28d99a6616a86942c3914476bf11997242a";
+	FileIndex *fi = new FileIndex("DUMB ID");
+
+	// loading fileindex
+	std::cout << std::endl << "Test load" << std::endl;
+	fi->loadIndex(testfile, fhash, 1532);
+	fi->printFileIndex(std::cout);
+	std::cout << "FileIndex Loaded" << std::endl << std::endl;
+
+	std::list<FileEntry *> hashresult;
+	std::list<FileEntry *> termresult;
+
+	// searchhash
+	std::string findhash = "82bffa6e1cdf8419397311789391238174817481";
+
+
+	std::cout << "Search hash : " << findhash << std::endl;
+	fi->searchHash(findhash, hashresult);
+
+	while(!hashresult.empty())
+	{
+		hashresult.back()->print(std::cout);
+		hashresult.pop_back();
+	}
+
+	// searchterm
+	std::list<std::string> terms;
+	terms.push_back("paper");
+	terms.push_back("doc");
+
+	std::cout << "Search terms" << std::endl;
+	fi->searchTerms(terms, termresult);
+
+	while(!termresult.empty())
+	{
+		termresult.back()->print(std::cout);
+		termresult.pop_back();
+	}
+
+	delete fi;
+	return 1;
+}