Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-02-02 10:35:15 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

removed dbase directory

Browse Source
This commit is contained in:
mr-alice 2016-09-21 22:27:53 +02:00
parent 0757fb303c
commit 0f5cf56cc6
9 changed files with 0 additions and 6142 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1323
libretroshare/src/dbase/cachestrapper.cc
View File

File diff suppressed because it is too large Load Diff

475
libretroshare/src/dbase/cachestrapper.h
View File

@ -1,475 +0,0 @@
/*
* RetroShare FileCache Module: cachestrapper.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_CACHE_STRAPPER_H
#define MRK_CACHE_STRAPPER_H
#include "pqi/p3cfgmgr.h"
#include "pqi/pqiservicemonitor.h"
#include "util/rsthreads.h"
#include <list>
#include <set>
#include <map>
#include <string>
#include <iostream>
/******************* CacheStrapper and Related Classes *******************
* A generic Cache Update system.
*
* CacheStrapper: maintains a set of CacheSources, and CacheStores,
* queries and updates as new information arrives.
*
* CacheTransfer: Interface for FileTransfer Class to support.
*
* CacheSource: Base Class for cache data provider. eg. FileIndexMonitor.
* CacheStore: Base Class for data cache. eg. FileCache/Store.
*
* Still TODO:
* (1) Design and Implement the Upload side of CacheTransfer/CacheStrapper.
* (2) CacheStrapper:: Save / Load Cache lists....
* (3) Clean up lists, maps on shutdown etc.
* (4) Consider Mutexes for multithreaded operations.
* (5) Test the MultiSource/Store capabilities.
*
******************* CacheStrapper and Related Classes *******************/
class CacheTransfer; /* Interface for File Transfer */
class CacheSource; /* Interface for local File Index/Monitor */
class CacheStore; /* Interface for the actual Cache */
class CacheStrapper; /* Controlling Class */
/******************************** CacheId ********************************/
/*!
* Use this to identify the type of cache source, strapper,
*/
class CacheId
{
public:
CacheId() :type(0), subid(0) { return; }
CacheId(uint16_t a, uint16_t b) :type(a), subid(b) { return; }
uint16_t type; /// cache types, this should be set to services type
uint16_t subid; /// should be initialised when using multicache feature of cachestrapper
};
bool operator<(const CacheId &a, const CacheId &b);
/*!
* Use for identifying physical files that have been chosen as cache data
* note: this does not actual store the data but serves to locate on network (via hash attribute,
* and on file via path)
*/
class RsCacheData
{
public:
RsPeerId pid; /// peer id
/// REMOVED as a WASTE to look it up everywhere! std::string pname; /// peer name (can be used by cachestore)
CacheId cid; /// cache id
std::string path; /// file system path where physical cache data is located
std::string name;
RsFileHash hash;
uint64_t size;
time_t recvd; /// received timestamp
};
std::ostream &operator<<(std::ostream &out, const RsCacheData &d);
/***************************** CacheTransfer *****************************/
/**
* Interface for FileTransfer Class to support cache
*/
class CacheTransfer
{
public:
CacheTransfer(CacheStrapper *cs) :strapper(cs) { return; }
virtual ~CacheTransfer() {}
/*!
* upload side of things .... searches through CacheStrapper.
*/
bool FindCacheFile(const RsFileHash& hash, std::string &path, uint64_t &size);
/*!
* At the download side RequestCache() => overloaded RequestCacheFile()
* the class should then call CompletedCache() or FailedCache()
*/
bool RequestCache(RsCacheData &data, CacheStore *cbStore); /* request from CacheStore */
protected:
/*!
* to be overloaded
*/
virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
bool CompletedCache(const RsFileHash &hash); /* internal completion -> does cb */
bool FailedCache(const RsFileHash &hash); /* internal completion -> does cb */
private:
CacheStrapper *strapper;
std::map<RsFileHash, RsCacheData> cbData;
std::map<RsFileHash, CacheStore *> cbStores;
};
/************************ CacheSource/CacheStore *************************/
typedef std::map<uint16_t, RsCacheData> CacheSet;
/*!
* Implements features needed for a service to act as a cachesource and allow pushing a of cache data from service to strapper
* Service is able to use this class for refresh its cache (push cache data)
* and interface to load and check cache availablility among peers (source of cache data)
* Architecturally Cachestrapper maintains the cachesource (which is passed as a pointer handle) while the cachesource-inheriting
* service can update cachesource as to new cache sources (cache data) created. Equivalently it enquiries through cache source for
* new cache data from peers
* @see p3Distrib
*/
class CacheSource
{
public:
CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir);
virtual ~CacheSource() {}
/*!
* called to determine available cache for peer -
* default acceptable (returns all)
*/
virtual bool cachesAvailable(const RsPeerId& pid, std::map<CacheId, RsCacheData> &ids)=0;
/*!
* function called at startup to load from
* configuration file....
* to be overloaded by inherited class
*/
virtual bool loadLocalCache(const RsCacheData &data);
/* control Caches available */
bool refreshCache(const RsCacheData &data,const std::set<RsPeerId>& destination_peers);
bool refreshCache(const RsCacheData &data);
bool clearCache(CacheId id);
/* controls if peer is an accepted receiver for cache items. Default is yes. To be overloaded. */
virtual bool isPeerAcceptedAsCacheReceiver(const RsPeerId& /*peer_id*/) { return true ; }
/* get private data */
std::string getCacheDir() { return cacheDir; }
bool isMultiCache() { return multiCache; }
uint16_t getCacheType() { return cacheType; }
/* display */
void listCaches(std::ostream &out);
/* search */
bool findCache(const RsFileHash& hash, RsCacheData &data) const;
protected:
uint16_t cacheType; /// for checking of cache type (usually of child class of source)
bool multiCache; /// whether multisource is in use or not.
CacheStrapper *mStrapper;
/*** MUTEX LOCKING */
void lockData() const;
void unlockData() const;
CacheSet caches; /// all local cache data stored here
std::map<RsFileHash, RsCacheData> mOldCaches; /// replaced/cleared caches are pushed here (in case requested)
private:
std::string cacheDir;
mutable RsMutex cMutex;
};
/*!
* Base Class for data cache. eg. FileCache/Store.
* This is best used to deal with external caches from other peers
* @see p3Distrib. pqiMonitor
*/
class CacheStore
{
public:
/*!
*
* @param t set to particular rs_service id. see rsserviceids.h
* @param m whether this is multicache service (true) or not(false)
* @param cs cache strapper instance responsible for maintaining the cache service
* @param cft cache transfer instance responsible for rquestiing and tranfering caches
* @param cachedir directory used to store cache related info for cachestore client
* @return
*/
CacheStore(uint16_t t, bool m, CacheStrapper *cs, CacheTransfer *cft, std::string cachedir);
virtual ~CacheStore() {}
/* current stored data */
/*!
*
* @param data returns cache data for pid/cid set in data itself
* @return false is unsuccessful and vice versa
*/
bool getStoredCache(RsCacheData &data); /* use pid/cid in data */
/*!
*
* @param data all cache store by cachestore is store here
* @return false not returned, only true at the moment
*/
bool getAllStoredCaches(std::list<RsCacheData> &data); /* use pid/cid in data */
/*!
* input from CacheStrapper -> store can then download new data
*/
void availableCache(const RsCacheData &data);
/*!
* should be called when the download is completed ... cache data is loaded
*/
void downloadedCache(const RsCacheData &data);
/*!
* called if the download fails, TODO: nothing done yet
*/
void failedCache(const RsCacheData &data);
/* virtual functions overloaded by cache implementor */
/* controls if peer is an accepted provider for cache items. Default is yes. To be overloaded. */
virtual bool isPeerAcceptedAsCacheProvider(const RsPeerId& /*peer_id*/) { return true ; }
/*!
* @param data cache data is stored here
* @return false is failed (cache does not exist), otherwise true
*/
virtual bool fetchCache(const RsCacheData &data); /* a question? */
virtual int nameCache(RsCacheData &data); /* fill in the name/path */
virtual int loadCache(const RsCacheData &data); /* actual load, once data available */
/* get private data */
std::string getCacheDir() { return cacheDir; }
bool isMultiCache() { return multiCache; }
uint16_t getCacheType() { return cacheType; }
/*!
* display, e.g. can pass std::out, cerr, ofstream, etc
*/
void listCaches(std::ostream &out);
protected:
/*!
* ** MUTEX LOCKING
*/
void lockData() const;
/*!
* ** MUTEX LOCKING
*/
void unlockData() const;
/*! This function is called to store Cache Entry in the CacheStore Table.
* it must be called from within a Mutex Lock....
*
* It doesn't lock itself -> to avoid race conditions
*/
void locked_storeCacheEntry(const RsCacheData &data);
/*! This function is called to store Cache Entry in the CacheStore Table.
* it must be called from within a Mutex Lock....
*
* It doesn't lock itself -> to avoid race conditions
*/
bool locked_getStoredCache(RsCacheData &data);
private:
uint16_t cacheType; /* for checking */
bool multiCache; /* do we care about subid's */
CacheStrapper *mStrapper;
CacheTransfer *cacheTransfer;
std::string cacheDir;
mutable RsMutex cMutex;
std::map<RsPeerId, CacheSet> caches;
};
/***************************** CacheStrapper *****************************/
/*!
* a convenient to pass cache handles to cachestrapper to maintain the cache service
* Make Sure you get the Ids right! see rsservicesids.h.
* When creating a cache service this data structure is
* source, usually the child class of store and source also serves as both handles
* @see CacheStrapper
*/
class CachePair
{
public:
/*!
* Default constructor, all variables set to NULL
*/
CachePair()
:source(NULL), store(NULL), id(0, 0) { return; }
/*!
*
* @param a cache source for service
* @param b cache store for service
* @param c the cache service id, c.type should be set to service id service-child class of store and source
*/
CachePair(CacheSource *a, CacheStore *b, CacheId c)
:source(a), store(b), id(c) { return; }
CacheSource *source;
CacheStore *store;
CacheId id; /// should be set id type to service types of store and source service-child class, and subid for multicache use
};
bool operator<(const CachePair &a, const CachePair &b);
/*!
* CacheStrapper: maintains a set of CacheSources, and CacheStores,
* queries and updates as new information arrives.
*/
class p3ServiceControl;
class CacheStrapper: public pqiServiceMonitor, public p3Config
{
public:
/*!
* @param cm handle used by strapper for getting peer connection information (online peers, sslids...)
* @return
*/
CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId);
virtual ~CacheStrapper() { return; }
/************* from pqiMonitor *******************/
virtual void statusChange(const std::list<pqiServicePeer> &plist);
/************* from pqiMonitor *******************/
/* Feedback from CacheSources */
/*!
* send data to peers online and self
* @param data
*
*/
void refreshCache(const RsCacheData &data);
void refreshCache(const RsCacheData &data,const std::set<RsPeerId>& destination_peers); // specify a particular list of destination peers (self not added!)
/*!
* forces config savelist
* @param data
* @see saveList()
*/
void refreshCacheStore(const RsCacheData &data);
/*!
* list of Caches to send out
*/
bool getCacheUpdates(std::list<std::pair<RsPeerId, RsCacheData> > &updates);
/*!
* add to strapper's cachepair set so a related service's store and source can be maintained
* @param pair the source and store handle for a service
*/
void addCachePair(CachePair pair);
/*** I/O (2) ***/
void recvCacheResponse(RsCacheData &data, time_t ts);
void handleCacheQuery(const RsPeerId& id, std::map<CacheId, RsCacheData> &data);
/*!
* search through CacheSources.
* @return false if cachedate mapping to hash not found
*/
bool findCache(const RsFileHash &hash, RsCacheData &data) const;
/* display */
void listCaches(std::ostream &out);
/*!
* does not do anything
* @param out
* @deprecated
*/
void listPeerStatus(std::ostream &out);
/**
* Checks if the cache physically exist at path given
* @param data
* @return whether it exists or not
*/
bool CacheExist(RsCacheData& data);
/* Config */
protected:
/* Key Functions to be overloaded for Full Configuration */
virtual RsSerialiser *setupSerialiser();
virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
virtual bool loadList(std::list<RsItem *>& load);
private:
/* these are static - so shouldn't need mutex */
p3ServiceControl *mServiceCtrl;
uint32_t mFtServiceId;
std::map<uint16_t, CachePair> caches;
RsMutex csMtx; /* protect below */
std::list<std::pair<RsPeerId, RsCacheData> > mCacheUpdates;
};
#endif

63
libretroshare/src/dbase/cachetest.h
View File

@ -1,63 +0,0 @@
/*
* RetroShare FileCache Module: cachetest.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_TEST_CACHE_H
#define MRK_TEST_CACHE_H
#include "cachestrapper.h"
#define TESTID 0xffff
#define TESTID2 0xffee
class CacheTestSource: public CacheSource
{
public:
CacheTestSource(CacheStrapper *cs, std::string dir)
:CacheSource(cs, TESTID, false, dir) { return; }
};
class CacheTestStore: public CacheStore
{
public:
CacheTestStore(CacheTransfer *cft, std::string dir)
:CacheStore(TESTID, false, cft, dir) { return; }
};
class CacheTestMultiSource: public CacheSource
{
public:
CacheTestMultiSource(CacheStrapper *cs, std::string dir)
:CacheSource(cs, TESTID2, true, dir) { return; }
};
class CacheTestMultiStore: public CacheStore
{
public:
CacheTestMultiStore(CacheTransfer *cft, std::string dir)
:CacheStore(TESTID2, true, cft, dir) { return; }
};
#endif

1706
libretroshare/src/dbase/fimonitor.cc
View File

File diff suppressed because it is too large Load Diff

220
libretroshare/src/dbase/fimonitor.h
View File

@ -1,220 +0,0 @@
/*
* RetroShare FileCache Module: fimonitor.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef FILE_INDEX_MONITOR_H
#define FILE_INDEX_MONITOR_H
#include "dbase/cachestrapper.h"
#include "dbase/findex.h"
#include "util/rsthreads.h"
#include "retroshare/rsfiles.h"
/******************************************************************************************
* The Local Monitoring Class: FileIndexMonitor.
*
* This periodically scans the directory tree, and updates any modified directories/files.
*
*****************************************************************************************/
/******************************************************************************************
STILL TODO:
(1) Implement Hash function.
bool FileIndexMonitor::hashFile(std::string path, FileEntry &fi);
(2) Add Shared directory controls to Monitor.
int FileIndexMonitor::addSharedDirectory(std::path);
int FileIndexMonitor::removeSharedDirectory(std::path);
std::string FileIndexMonitor::findRealRoot(std::string base);
These must be split into <base>/<top> and the mapping saved.
eg: addSharedDirectory("c:/home/stuff/dir1") --> "c:/home/stuff" <-> "dir1"
This code has been written already, and can just be moved over.
FOR LATER:
(2) Port File/Directory lookup code to windoze. (or compile dirent.c under windoze)
(3) Add Load/Store interface to FileIndexMonitor. (later)
(4) Integrate with real Thread/Mutex code (last thing to do)
******************************************************************************************/
class DirContentToHash
{
public:
std::vector<FileEntry> fentries ;
std::string realpath ;
std::string dirpath ;
};
class HashCache
{
public:
HashCache(const std::string& save_file_name) ;
void save() ;
void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ;
bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ;
void clean() ;
typedef struct
{
uint64_t size ;
uint64_t time_stamp ;
uint64_t modf_stamp ;
RsFileHash hash ;
} HashCacheInfo ;
void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; }
uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; }
void clear() { _files.clear(); }
bool empty() const { return _files.empty() ; }
private:
uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries
std::map<std::string, HashCacheInfo> _files ;
std::string _path ;
bool _changed ;
};
/******************************************************************************************
* FileIndexMonitor
*****************************************************************************************/
class FileIndexMonitor: public CacheSource, public RsTickingThread
{
public:
FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid, const std::string& config_dir);
virtual ~FileIndexMonitor();
/* external interface for filetransfer */
bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list<RsNodeGroupId>& parent_groups) const;
int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) ;
int SearchBoolExp(Expression *exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
int filterResults(std::list<FileEntry*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
/* external interface for local access to files */
bool convertSharedFilePath(std::string path, std::string &fullpath);
/* Interacting with CacheSource */
/* overloaded from CacheSource */
virtual bool loadLocalCache(const RsCacheData &data); /* called with stored data */
bool updateCache(const RsCacheData &data,const std::set<RsPeerId>& dest_peers); /* we call when we have a new cache for others */
/* the FileIndexMonitor inner workings */
//virtual void run(std::string& currentJob); /* overloaded from RsThread */
//void updateCycle(std::string& currentJob);
virtual void data_tick(); /* overloaded from RsThread */
void updateCycle();
// Interface for browsing dir hirarchy
int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ;
uint32_t getType(void*) const ;
int RequestDirDetails(const std::string& path, DirDetails &details) const ;
// set/update shared directories
virtual void setSharedDirectories(const std::list<SharedDirInfo>& dirs);
void getSharedDirectories(std::list<SharedDirInfo>& dirs);
void updateShareFlags(const SharedDirInfo& info) ;
void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed
void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files.
bool inDirectoryCheck();
/* util fns */
// from CacheSource
virtual bool cachesAvailable(const RsPeerId& pid, std::map<CacheId, RsCacheData> &ids) ;
protected:
// Sets/gets the duration period within which already hashed files are remembered.
//
void setRememberHashFilesDuration(uint32_t days) ;
uint32_t rememberHashFilesDuration() const ;
void setRememberHashFiles(bool) ;
bool rememberHashFiles() ;
// Remove any memory of formerly hashed files that are not shared anymore
void clearHashFiles() ;
void setPeriod(int insecs);
int getPeriod() const;
bool autoCheckEnabled() const ;
private:
/* the mutex should be locked before calling these 3. */
// Saves file indexs and update the cache. Returns the name of the main
// file index, which becomes the new reference file for mod times.
//
time_t locked_saveFileIndexes(bool update_cache) ;
// Finds the share flags associated with this file entry.
void locked_findShareFlagsAndParentGroups(FileEntry *fe, FileStorageFlags& shareflags, std::list<RsNodeGroupId> &parent_groups) const ;
std::string locked_findRealRoot(std::string base) const;
void hashFiles(const std::vector<DirContentToHash>& to_hash) ;
bool hashFile(std::string path, FileEntry &fi); /* To Implement */
/* data */
mutable RsMutex fiMutex;
FileIndex fi;
int updatePeriod;
std::map<std::string, SharedDirInfo> directoryMap; /* used by findRealRoot */
/* flags to kick - if we were busy or sleeping */
bool pendingDirs;
bool pendingForceCacheWrite;
/* flags to force Check, to tell if we're in check */
bool mForceCheck;
bool mInCheck;
std::list<SharedDirInfo> pendingDirList;
bool internal_setSharedDirectories();
HashCache hashCache ;
bool useHashCache ;
std::map<RsPeerId,RsCacheData> _cache_items_per_peer ; // stored the cache items to be sent to each peer.
// This file is the location of the current index file. When checking for new files, we compare the modification time
// of this file to the mod time of the files on the disk. This allows to now account for time-shift in the computer.
//
time_t reference_time ;
};
#endif

1516
libretroshare/src/dbase/findex.cc
View File

File diff suppressed because it is too large Load Diff

278
libretroshare/src/dbase/findex.h
View File

@ -1,278 +0,0 @@
/*
* RetroShare FileCache Module: findex.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef FILE_INDEX_H
#define FILE_INDEX_H
#include <string>
#include <map>
#include <set>
#if __MACH__
#include <unordered_set>
#else
#include <tr1/unordered_set>
#endif
#include <list>
#include <vector>
#include <stdint.h>
#include "retroshare/rstypes.h"
/******************************************************************************************
* The Key Data Types for the File Index:
FileEntry : Information about a single file.
DirEntry : Information about a directory and its children
PersonEntry : Information the root of a FileIndex.
FileIndex : A Collection of root directories, with a set of functions to manipulate them.
In terms of retroshare, There will be a single 'Local' FileIndex used to store
the shared files, and a set of 'Remote' FileIndices which are used to store
the available files of all the peers.
******************************************************************************************/
/******************************************************************************************
STILL TODO:
(1) Load/Store a FileIndex to file...
int FileIndex::loadIndex(FILE *input);
int FileIndex::saveIndex(FILE *input);
This can be done in a recursive manner, or handled completely within FileIndex.
(2) Search Functions for Partial File Names and Hashes.
int FileIndex::searchHash(std::string hash, std::list<FileEntry> &results);
int FileIndex::searchTerms(std::list<string> terms, std::list<FileEntry> &results);
This is probably best done in a recursive manner.
The search could also be extended to handle complex Boolean searches such as :
match (size > 100K) && (name contains 'Blue') .... if anyone is interested.
But this can get quite complicated, and can be left to a later date.
******************************************************************************************/
/******************************************************************************************
* FileEntry
*****************************************************************************************/
#include <util/smallobject.h>
class DirEntry;
class FileEntry: public RsMemoryManagement::SmallObject
{
public:
FileEntry()
: size(0), modtime(0), pop(0), updtime(0), parent(NULL), row(0)
{ return; }
virtual ~FileEntry() { return; }
virtual uint32_t type() const { return DIR_TYPE_FILE ; }
virtual int print(std::string &out);
/* Data */
std::string name;
RsFileHash hash;
uint64_t size; /* file size */
time_t modtime; /* modification time - most recent mod time for a sub entry for dirs */
int pop; /* popularity rating */
time_t updtime; /* last updated */
/* References for easy manipulation */
DirEntry *parent;
int row;
std::list<std::string> parent_groups ;
};
/******************************************************************************************
* DirEntry
*****************************************************************************************/
class DirEntry: public FileEntry
{
public:
DirEntry() : most_recent_time(0) {}
/* cleanup */
virtual ~DirEntry();
/* update local entries */
DirEntry * updateDir(const FileEntry& fe, time_t updtime);
FileEntry * updateFile(const FileEntry& fe, time_t updtime);
virtual uint32_t type() const { return DIR_TYPE_DIR ; }
int checkParentPointers();
int updateChildRows();
/* remove local entries */
int removeFile(const std::string& name);
int removeDir(const std::string& name);
int removeOldDir(const std::string& name, time_t old); /* checks ts first */
/* recursive cleanup */
int removeOldEntries(time_t old, bool recursive);
/* recursive searches */
DirEntry * findOldDirectory(time_t old);
DirEntry * findDirectory(const std::string& path);
/* recursive update directory mod/pop values */
int updateDirectories(const std::string& path, int pop, int modtime);
/* output */
int print(std::string &out);
int saveEntry(std::string &out);
void writeDirInfo(std::string&);
void writeFileInfo(std::string&);
/* Data */
std::string path; /* full path (includes name) */
std::map<std::string, DirEntry *> subdirs;
std::map<std::string, FileEntry *> files;
time_t most_recent_time; /* last updated */
/* Inherited members from FileEntry:
int size - count for dirs
std::string name; - directory name
std::string hash; - not used
int size; - not used
int modtime; - most recent modication time of any child file (recursive)
int pop; - most popular child file (recursive)
int updtime; - last updated
*/
};
/******************************************************************************************
* PersonEntry
*****************************************************************************************/
class PersonEntry: public DirEntry
{
public:
/* cleanup */
PersonEntry(const RsPeerId& pid) : id(pid) { return; }
virtual ~PersonEntry() { return; }
DirEntry &operator=(DirEntry &src)
{
DirEntry *pdest = this;
(*pdest) = src;
return *this;
}
virtual uint32_t type() const { return DIR_TYPE_PERSON ; }
/* Data */
RsPeerId id;
/* Inherited members from FileEntry:
int size - count for dirs
std::string name; - directory name
std::string hash; - not used
int size; - not used
int modtime; - most recent modication time of any child file (recursive)
int pop; - most popular child file (recursive)
int updtime; - last updated
*/
};
/******************************************************************************************
* FileIndex
*****************************************************************************************/
class Expression;
class FileIndex
{
public:
FileIndex(const RsPeerId& pid);
~FileIndex();
/* control root entries */
int setRootDirectories(const std::list<std::string> &inlist, time_t utime);
int getRootDirectories(std::list<std::string> &outlist);
/* update (index building) */
DirEntry * updateDirEntry(const std::string& path, const FileEntry& fe, time_t utime);
FileEntry * updateFileEntry(const std::string& path, const FileEntry& fe, time_t utime);
DirEntry * findOldDirectory(time_t old); /* finds directories older than old */
int removeOldDirectory(const std::string& fpath, const std::string& name, time_t old);
int cleanOldEntries(time_t old); /* removes entries older than old */
/* debug */
int printFileIndex(std::string &out);
int printFileIndex(std::ostream &out);
/* load/save to file */
int loadIndex(const std::string& filename, const RsFileHash &expectedHash, uint64_t size);
int saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size, const std::set<std::string>& forbidden_roots);
/* search through this index */
int searchTerms(const std::list<std::string>& terms, std::list<FileEntry *> &results) const;
int searchHash(const RsFileHash& hash, std::list<FileEntry *> &results) const;
int searchBoolExp(Expression * exp, std::list<FileEntry *> &results) const;
/// Recursively compute the maximum modification time of children.
/// Used to easily retrieve mose recent files.
//
void updateMaxModTime() ;
void RecursUpdateMaxModTime(DirEntry *) ;
PersonEntry *root;
#ifdef __MACH__
static std::unordered_set<void*> _pointers ;
#else
static std::tr1::unordered_set<void*> _pointers ;
#endif
static void registerEntry(void*p) ;
static void unregisterEntry(void*p) ;
static bool isValid(void*p) ;
/// Fills up details from the data contained in ref.
//
static bool extractData(void *ref,DirDetails& details) ;
static uint32_t getType(void *ref) ;
void *findRef(const std::string& path) const ;
bool extractData(const std::string& path,DirDetails& details) const ;
void updateHashIndex() ;
void recursUpdateHashIndex(DirEntry *) ;
std::map<RsFileHash,FileEntry*> _file_hashes ;
};
#endif

459
libretroshare/src/dbase/fistore.cc
View File

@ -1,459 +0,0 @@
/*
* RetroShare FileCache Module: fistore.cc
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include <time.h>
#include "rsserver/p3face.h"
#include "dbase/fistore.h"
#include "retroshare/rsexpr.h"
#include "retroshare/rsfiles.h"
#include "serialiser/rsserviceids.h"
#include "pqi/p3peermgr.h"
#include "pqi/p3notify.h"
FileIndexStore::FileIndexStore(CacheStrapper *cs, CacheTransfer *cft,
p3PeerMgr *cnmgr, RsPeerId ownid, std::string cachedir)
:CacheStore(RS_SERVICE_TYPE_FILE_INDEX, false, cs, cft, cachedir),
localId(ownid), localindex(NULL), mPeerMgr(cnmgr)
{
return;
}
FileIndexStore::~FileIndexStore()
{
/* clean up the Index */
return;
}
/***
* #define FIS_DEBUG2 1
* #define FIS_DEBUG 1
**/
/* actual load, once data available */
int FileIndexStore::loadCache(const RsCacheData &data)
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() hash: " << data.hash << std::endl;
std::cerr << "FileIndexStore::loadCache() path: " << data.path << std::endl;
std::cerr << "FileIndexStore::loadCache() name: " << data.name << std::endl;
std::cerr << "FileIndexStore::loadCache() size: " << data.size << std::endl;
#endif
/* do Callback */
AboutToModify();
/* lock it up */
lockData();
FileIndex *fiold = NULL;
bool local = (data.pid == localId);
std::map<RsPeerId, FileIndex *>::iterator it;
/* remove old cache */
if (local)
{
fiold = localindex;
localindex = NULL;
}
else if (indices.end() != (it = indices.find(data.pid)))
{
fiold = it->second;
indices.erase(it);
//delete fi;
}
if(mPeerMgr->isFriend(data.pid))
{
// We discard file lists from non friends. This is the place to remove file lists of deleted friends
// from the cache. Doing this, the file list still shows in a session where we deleted a friend, but will be removed
// at next restart.
//
/* load Cache */
FileIndex *finew = new FileIndex(data.pid);
if (finew->loadIndex(data.path + '/' + data.name, data.hash, data.size))
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() Succeeded!" << std::endl;
#endif
/* This is not the place to set the peername.
* It is a hack, which makes it programmatically impossible
* to get the file data out....
*
* peername should not be used in dbase.
*/
finew->root->name = data.pid.toStdString();
if (local)
{
localindex = finew;
}
else
{
indices[data.pid] = finew;
}
delete fiold;
/* store in tale */
locked_storeCacheEntry(data);
}
else
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() Failed!" << std::endl;
#endif
/* reinstall the old one! */
delete finew;
if (fiold)
{
if (local)
{
localindex = fiold;
}
else
{
indices[data.pid] = fiold;
}
}
}
}
#ifdef FIS_DEBUG
else
std::cerr << "Discarding file list from deleted peer " << data.pid << std::endl ;
#endif
/* need to correct indices(row) for the roots of the FileIndex */
int i = 0;
for(it = indices.begin(); it != indices.end(); ++it)
{
(it->second)->root->row = i++;
it->second->FileIndex::updateMaxModTime() ;
}
if (localindex)
{
localindex->root->row = 0;
localindex->updateMaxModTime() ;
}
unlockData();
ModCompleted();
bool ret = false;
return ret;
}
/* Search Interface - For Directory Access */
int FileIndexStore::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails& details) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator it = indices.find(uid);
bool found = true;
if (it != indices.end())
found = it->second->extractData(path,details) ;
unlockData();
return found ;
#ifdef OLD_STUFF_TO_REMOVE
/* lock it up */
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator it;
it = indices.find(uid);
bool found = true;
if (it != indices.end())
{
//DirEntry *fdir = (it->second).lookupDirectory(path);
/* translate it
*/
bool b = FileIndex::extractData((it->second)->root,details) ;
found = found && b ;
}
else
found = false;
unlockData();
return found;
#endif
}
int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const
{
/* remove unused parameter warnings */
(void) flags;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl;
#endif
std::map<RsPeerId, FileIndex *>::const_iterator pit;
lockData();
// checked by FileIndex::extractData
// if(ref != NULL && !FileIndex::isValid(ref))
// {
// unlockData() ;
// return false ;
// }
/* so cast *ref to a DirEntry */
/* root case */
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() CHKS" << std::endl;
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
(pit->second)->root->checkParentPointers();
}
#endif
if (ref == NULL)
{
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() ref=NULL (root)" << std::endl;
#endif
/* get remote root entries */
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
/*
*/
FileIndex *fileIndex = pit->second;
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = fileIndex->root->name;
stub.ref = fileIndex->root;
details.children.push_back(stub);
}
details.parent = NULL;
details.prow = -1;
details.ref = NULL;
details.type = DIR_TYPE_ROOT;
details.name = "";
details.hash.clear() ;
details.path = "";
details.count = indices.size();
details.age = 0;
details.flags.clear() ;
details.min_age = 0;
unlockData();
return true ;
}
bool b = FileIndex::extractData(ref,details) ;
unlockData();
return b;
}
uint32_t FileIndexStore::getType(void *ref) const
{
lockData() ;
uint32_t b = FileIndex::getType(ref) ;
unlockData();
return b;
}
int FileIndexStore::SearchHash(const RsFileHash& hash, std::list<FileDetail> &results) const
{
lockData();
results.clear() ;
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
time_t now = time(NULL);
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash()" << std::endl;
#endif
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash() Searching: Peer ";
std::cerr << pit->first << std::endl;
#endif
firesults.clear();
(pit->second)->searchHash(hash, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
FileDetail fd;
fd.id = pit->first;
fd.name = (*rit)->name;
fd.hash = (*rit)->hash;
fd.path = ""; /* TODO */
fd.size = (*rit)->size;
fd.age = now - (*rit)->modtime;
fd.rank = (*rit)->pop;
results.push_back(fd);
}
}
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash() Found " << results.size();
std::cerr << " Results from " << indices.size() << " Peers" << std::endl;
#endif
unlockData();
return results.size();
}
int FileIndexStore::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
results.clear() ;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchKeywords()" << std::endl;
#endif
if(flags & RS_FILE_HINTS_REMOTE)
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
firesults.clear();
(pit->second)->searchTerms(keywords, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!FileIndex::extractData(*rit, dd))
continue ;
results.push_back(dd);
}
}
if(flags & RS_FILE_HINTS_LOCAL)
if (localindex)
{
firesults.clear();
localindex->searchTerms(keywords, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!FileIndex::extractData(*rit, dd))
continue ;
dd.id.clear() ;
results.push_back(dd);
}
}
unlockData();
return results.size();
}
int FileIndexStore::searchBoolExp(Expression * exp, std::list<DirDetails> &results) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::searchBoolExp()" << std::endl;
#endif
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
firesults.clear();
(pit->second)->searchBoolExp(exp, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
FileIndex::extractData(*rit, dd);
results.push_back(dd);
}
}
/* finally search local files */
if (localindex)
{
firesults.clear();
localindex->searchBoolExp(exp, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
FileIndex::extractData(*rit, dd);
dd.id.clear() ;
results.push_back(dd);
}
}
unlockData();
return results.size();
}
int FileIndexStore::AboutToModify()
{
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
return 1;
}
int FileIndexStore::ModCompleted()
{
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
return 1;
}

102
libretroshare/src/dbase/fistore.h
View File

@ -1,102 +0,0 @@
/*
* RetroShare FileCache Module: fistore.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_FILE_INDEX_STORE_H
#define MRK_FILE_INDEX_STORE_H
/**********
* Stores the FileCaches of the Peers
* must implement 'loadCache' to
*
* This class is also accessed by the GUI....
* and the FileTransfer class.
*
*/
class p3PeerMgr ;
#include "dbase/findex.h"
#include "dbase/cachestrapper.h"
#include "retroshare/rsiface.h"
class FileStoreResult
{
public:
std::string id;
std::string path;
std::string hash;
std::string name;
};
class NotifyCallback
{
public:
NotifyCallback() { return; }
virtual ~NotifyCallback() { return; }
virtual void AboutToModify() { return; }
virtual void ModCompleted() { return; }
};
class Expression;
class FileIndexStore: public CacheStore
{
public:
FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, p3PeerMgr *pmgr, RsPeerId ownid, std::string cachedir);
virtual ~FileIndexStore();
/* virtual functions overloaded by cache implementor */
virtual int loadCache(const RsCacheData &data); /* actual load, once data available */
/* Search Interface - For FileTransfer Lookup */
int SearchHash(const RsFileHash &hash, std::list<FileDetail> &results) const;
/* Search Interface - For Search Interface */
int SearchKeywords(std::list<std::string> terms, std::list<DirDetails> &results,FileSearchFlags flags) const;
/* Search Interface - for Adv Search Interface */
int searchBoolExp(Expression * exp, std::list<DirDetails> &results) const;
/* Search Interface - For Directory Access */
int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) const;
int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const;
uint32_t getType(void *ref) const ;
private:
int AboutToModify();
int ModCompleted();
std::map<RsPeerId, FileIndex *> indices;
RsPeerId localId;
FileIndex *localindex;
p3PeerMgr *mPeerMgr ;
};
#endif