Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-05-02 06:06:10 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

merged branch v0.6-FileLists locally into master

Browse source
This commit is contained in:
csoler 2016-09-24 14:47:45 +02:00
commit ee54c02101
80 changed files with 7400 additions and 7939 deletions
Download patch file Download diff file Expand all files Collapse all files

1323
libretroshare/src/dbase/cachestrapper.cc
View file

File diff suppressed because it is too large Load diff

475
libretroshare/src/dbase/cachestrapper.h
View file

@ -1,475 +0,0 @@
/*
* RetroShare FileCache Module: cachestrapper.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_CACHE_STRAPPER_H
#define MRK_CACHE_STRAPPER_H
#include "pqi/p3cfgmgr.h"
#include "pqi/pqiservicemonitor.h"
#include "util/rsthreads.h"
#include <list>
#include <set>
#include <map>
#include <string>
#include <iostream>
/******************* CacheStrapper and Related Classes *******************
* A generic Cache Update system.
*
* CacheStrapper: maintains a set of CacheSources, and CacheStores,
* queries and updates as new information arrives.
*
* CacheTransfer: Interface for FileTransfer Class to support.
*
* CacheSource: Base Class for cache data provider. eg. FileIndexMonitor.
* CacheStore: Base Class for data cache. eg. FileCache/Store.
*
* Still TODO:
* (1) Design and Implement the Upload side of CacheTransfer/CacheStrapper.
* (2) CacheStrapper:: Save / Load Cache lists....
* (3) Clean up lists, maps on shutdown etc.
* (4) Consider Mutexes for multithreaded operations.
* (5) Test the MultiSource/Store capabilities.
*
******************* CacheStrapper and Related Classes *******************/
class CacheTransfer; /* Interface for File Transfer */
class CacheSource; /* Interface for local File Index/Monitor */
class CacheStore; /* Interface for the actual Cache */
class CacheStrapper; /* Controlling Class */
/******************************** CacheId ********************************/
/*!
* Use this to identify the type of cache source, strapper,
*/
class CacheId
{
public:
CacheId() :type(0), subid(0) { return; }
CacheId(uint16_t a, uint16_t b) :type(a), subid(b) { return; }
uint16_t type; /// cache types, this should be set to services type
uint16_t subid; /// should be initialised when using multicache feature of cachestrapper
};
bool operator<(const CacheId &a, const CacheId &b);
/*!
* Use for identifying physical files that have been chosen as cache data
* note: this does not actual store the data but serves to locate on network (via hash attribute,
* and on file via path)
*/
class RsCacheData
{
public:
RsPeerId pid; /// peer id
/// REMOVED as a WASTE to look it up everywhere! std::string pname; /// peer name (can be used by cachestore)
CacheId cid; /// cache id
std::string path; /// file system path where physical cache data is located
std::string name;
RsFileHash hash;
uint64_t size;
time_t recvd; /// received timestamp
};
std::ostream &operator<<(std::ostream &out, const RsCacheData &d);
/***************************** CacheTransfer *****************************/
/**
* Interface for FileTransfer Class to support cache
*/
class CacheTransfer
{
public:
CacheTransfer(CacheStrapper *cs) :strapper(cs) { return; }
virtual ~CacheTransfer() {}
/*!
* upload side of things .... searches through CacheStrapper.
*/
bool FindCacheFile(const RsFileHash& hash, std::string &path, uint64_t &size);
/*!
* At the download side RequestCache() => overloaded RequestCacheFile()
* the class should then call CompletedCache() or FailedCache()
*/
bool RequestCache(RsCacheData &data, CacheStore *cbStore); /* request from CacheStore */
protected:
/*!
* to be overloaded
*/
virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
bool CompletedCache(const RsFileHash &hash); /* internal completion -> does cb */
bool FailedCache(const RsFileHash &hash); /* internal completion -> does cb */
private:
CacheStrapper *strapper;
std::map<RsFileHash, RsCacheData> cbData;
std::map<RsFileHash, CacheStore *> cbStores;
};
/************************ CacheSource/CacheStore *************************/
typedef std::map<uint16_t, RsCacheData> CacheSet;
/*!
* Implements features needed for a service to act as a cachesource and allow pushing a of cache data from service to strapper
* Service is able to use this class for refresh its cache (push cache data)
* and interface to load and check cache availablility among peers (source of cache data)
* Architecturally Cachestrapper maintains the cachesource (which is passed as a pointer handle) while the cachesource-inheriting
* service can update cachesource as to new cache sources (cache data) created. Equivalently it enquiries through cache source for
* new cache data from peers
* @see p3Distrib
*/
class CacheSource
{
public:
CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir);
virtual ~CacheSource() {}
/*!
* called to determine available cache for peer -
* default acceptable (returns all)
*/
virtual bool cachesAvailable(const RsPeerId& pid, std::map<CacheId, RsCacheData> &ids)=0;
/*!
* function called at startup to load from
* configuration file....
* to be overloaded by inherited class
*/
virtual bool loadLocalCache(const RsCacheData &data);
/* control Caches available */
bool refreshCache(const RsCacheData &data,const std::set<RsPeerId>& destination_peers);
bool refreshCache(const RsCacheData &data);
bool clearCache(CacheId id);
/* controls if peer is an accepted receiver for cache items. Default is yes. To be overloaded. */
virtual bool isPeerAcceptedAsCacheReceiver(const RsPeerId& /*peer_id*/) { return true ; }
/* get private data */
std::string getCacheDir() { return cacheDir; }
bool isMultiCache() { return multiCache; }
uint16_t getCacheType() { return cacheType; }
/* display */
void listCaches(std::ostream &out);
/* search */
bool findCache(const RsFileHash& hash, RsCacheData &data) const;
protected:
uint16_t cacheType; /// for checking of cache type (usually of child class of source)
bool multiCache; /// whether multisource is in use or not.
CacheStrapper *mStrapper;
/*** MUTEX LOCKING */
void lockData() const;
void unlockData() const;
CacheSet caches; /// all local cache data stored here
std::map<RsFileHash, RsCacheData> mOldCaches; /// replaced/cleared caches are pushed here (in case requested)
private:
std::string cacheDir;
mutable RsMutex cMutex;
};
/*!
* Base Class for data cache. eg. FileCache/Store.
* This is best used to deal with external caches from other peers
* @see p3Distrib. pqiMonitor
*/
class CacheStore
{
public:
/*!
*
* @param t set to particular rs_service id. see rsserviceids.h
* @param m whether this is multicache service (true) or not(false)
* @param cs cache strapper instance responsible for maintaining the cache service
* @param cft cache transfer instance responsible for rquestiing and tranfering caches
* @param cachedir directory used to store cache related info for cachestore client
* @return
*/
CacheStore(uint16_t t, bool m, CacheStrapper *cs, CacheTransfer *cft, std::string cachedir);
virtual ~CacheStore() {}
/* current stored data */
/*!
*
* @param data returns cache data for pid/cid set in data itself
* @return false is unsuccessful and vice versa
*/
bool getStoredCache(RsCacheData &data); /* use pid/cid in data */
/*!
*
* @param data all cache store by cachestore is store here
* @return false not returned, only true at the moment
*/
bool getAllStoredCaches(std::list<RsCacheData> &data); /* use pid/cid in data */
/*!
* input from CacheStrapper -> store can then download new data
*/
void availableCache(const RsCacheData &data);
/*!
* should be called when the download is completed ... cache data is loaded
*/
void downloadedCache(const RsCacheData &data);
/*!
* called if the download fails, TODO: nothing done yet
*/
void failedCache(const RsCacheData &data);
/* virtual functions overloaded by cache implementor */
/* controls if peer is an accepted provider for cache items. Default is yes. To be overloaded. */
virtual bool isPeerAcceptedAsCacheProvider(const RsPeerId& /*peer_id*/) { return true ; }
/*!
* @param data cache data is stored here
* @return false is failed (cache does not exist), otherwise true
*/
virtual bool fetchCache(const RsCacheData &data); /* a question? */
virtual int nameCache(RsCacheData &data); /* fill in the name/path */
virtual int loadCache(const RsCacheData &data); /* actual load, once data available */
/* get private data */
std::string getCacheDir() { return cacheDir; }
bool isMultiCache() { return multiCache; }
uint16_t getCacheType() { return cacheType; }
/*!
* display, e.g. can pass std::out, cerr, ofstream, etc
*/
void listCaches(std::ostream &out);
protected:
/*!
* ** MUTEX LOCKING
*/
void lockData() const;
/*!
* ** MUTEX LOCKING
*/
void unlockData() const;
/*! This function is called to store Cache Entry in the CacheStore Table.
* it must be called from within a Mutex Lock....
*
* It doesn't lock itself -> to avoid race conditions
*/
void locked_storeCacheEntry(const RsCacheData &data);
/*! This function is called to store Cache Entry in the CacheStore Table.
* it must be called from within a Mutex Lock....
*
* It doesn't lock itself -> to avoid race conditions
*/
bool locked_getStoredCache(RsCacheData &data);
private:
uint16_t cacheType; /* for checking */
bool multiCache; /* do we care about subid's */
CacheStrapper *mStrapper;
CacheTransfer *cacheTransfer;
std::string cacheDir;
mutable RsMutex cMutex;
std::map<RsPeerId, CacheSet> caches;
};
/***************************** CacheStrapper *****************************/
/*!
* a convenient to pass cache handles to cachestrapper to maintain the cache service
* Make Sure you get the Ids right! see rsservicesids.h.
* When creating a cache service this data structure is
* source, usually the child class of store and source also serves as both handles
* @see CacheStrapper
*/
class CachePair
{
public:
/*!
* Default constructor, all variables set to NULL
*/
CachePair()
:source(NULL), store(NULL), id(0, 0) { return; }
/*!
*
* @param a cache source for service
* @param b cache store for service
* @param c the cache service id, c.type should be set to service id service-child class of store and source
*/
CachePair(CacheSource *a, CacheStore *b, CacheId c)
:source(a), store(b), id(c) { return; }
CacheSource *source;
CacheStore *store;
CacheId id; /// should be set id type to service types of store and source service-child class, and subid for multicache use
};
bool operator<(const CachePair &a, const CachePair &b);
/*!
* CacheStrapper: maintains a set of CacheSources, and CacheStores,
* queries and updates as new information arrives.
*/
class p3ServiceControl;
class CacheStrapper: public pqiServiceMonitor, public p3Config
{
public:
/*!
* @param cm handle used by strapper for getting peer connection information (online peers, sslids...)
* @return
*/
CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId);
virtual ~CacheStrapper() { return; }
/************* from pqiMonitor *******************/
virtual void statusChange(const std::list<pqiServicePeer> &plist);
/************* from pqiMonitor *******************/
/* Feedback from CacheSources */
/*!
* send data to peers online and self
* @param data
*
*/
void refreshCache(const RsCacheData &data);
void refreshCache(const RsCacheData &data,const std::set<RsPeerId>& destination_peers); // specify a particular list of destination peers (self not added!)
/*!
* forces config savelist
* @param data
* @see saveList()
*/
void refreshCacheStore(const RsCacheData &data);
/*!
* list of Caches to send out
*/
bool getCacheUpdates(std::list<std::pair<RsPeerId, RsCacheData> > &updates);
/*!
* add to strapper's cachepair set so a related service's store and source can be maintained
* @param pair the source and store handle for a service
*/
void addCachePair(CachePair pair);
/*** I/O (2) ***/
void recvCacheResponse(RsCacheData &data, time_t ts);
void handleCacheQuery(const RsPeerId& id, std::map<CacheId, RsCacheData> &data);
/*!
* search through CacheSources.
* @return false if cachedate mapping to hash not found
*/
bool findCache(const RsFileHash &hash, RsCacheData &data) const;
/* display */
void listCaches(std::ostream &out);
/*!
* does not do anything
* @param out
* @deprecated
*/
void listPeerStatus(std::ostream &out);
/**
* Checks if the cache physically exist at path given
* @param data
* @return whether it exists or not
*/
bool CacheExist(RsCacheData& data);
/* Config */
protected:
/* Key Functions to be overloaded for Full Configuration */
virtual RsSerialiser *setupSerialiser();
virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
virtual bool loadList(std::list<RsItem *>& load);
private:
/* these are static - so shouldn't need mutex */
p3ServiceControl *mServiceCtrl;
uint32_t mFtServiceId;
std::map<uint16_t, CachePair> caches;
RsMutex csMtx; /* protect below */
std::list<std::pair<RsPeerId, RsCacheData> > mCacheUpdates;
};
#endif

63
libretroshare/src/dbase/cachetest.h
View file

@ -1,63 +0,0 @@
/*
* RetroShare FileCache Module: cachetest.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_TEST_CACHE_H
#define MRK_TEST_CACHE_H
#include "cachestrapper.h"
#define TESTID 0xffff
#define TESTID2 0xffee
class CacheTestSource: public CacheSource
{
public:
CacheTestSource(CacheStrapper *cs, std::string dir)
:CacheSource(cs, TESTID, false, dir) { return; }
};
class CacheTestStore: public CacheStore
{
public:
CacheTestStore(CacheTransfer *cft, std::string dir)
:CacheStore(TESTID, false, cft, dir) { return; }
};
class CacheTestMultiSource: public CacheSource
{
public:
CacheTestMultiSource(CacheStrapper *cs, std::string dir)
:CacheSource(cs, TESTID2, true, dir) { return; }
};
class CacheTestMultiStore: public CacheStore
{
public:
CacheTestMultiStore(CacheTransfer *cft, std::string dir)
:CacheStore(TESTID2, true, cft, dir) { return; }
};
#endif

1706
libretroshare/src/dbase/fimonitor.cc
View file

File diff suppressed because it is too large Load diff

220
libretroshare/src/dbase/fimonitor.h
View file

@ -1,220 +0,0 @@
/*
* RetroShare FileCache Module: fimonitor.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef FILE_INDEX_MONITOR_H
#define FILE_INDEX_MONITOR_H
#include "dbase/cachestrapper.h"
#include "dbase/findex.h"
#include "util/rsthreads.h"
#include "retroshare/rsfiles.h"
/******************************************************************************************
* The Local Monitoring Class: FileIndexMonitor.
*
* This periodically scans the directory tree, and updates any modified directories/files.
*
*****************************************************************************************/
/******************************************************************************************
STILL TODO:
(1) Implement Hash function.
bool FileIndexMonitor::hashFile(std::string path, FileEntry &fi);
(2) Add Shared directory controls to Monitor.
int FileIndexMonitor::addSharedDirectory(std::path);
int FileIndexMonitor::removeSharedDirectory(std::path);
std::string FileIndexMonitor::findRealRoot(std::string base);
These must be split into <base>/<top> and the mapping saved.
eg: addSharedDirectory("c:/home/stuff/dir1") --> "c:/home/stuff" <-> "dir1"
This code has been written already, and can just be moved over.
FOR LATER:
(2) Port File/Directory lookup code to windoze. (or compile dirent.c under windoze)
(3) Add Load/Store interface to FileIndexMonitor. (later)
(4) Integrate with real Thread/Mutex code (last thing to do)
******************************************************************************************/
class DirContentToHash
{
public:
std::vector<FileEntry> fentries ;
std::string realpath ;
std::string dirpath ;
};
class HashCache
{
public:
HashCache(const std::string& save_file_name) ;
void save() ;
void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ;
bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ;
void clean() ;
typedef struct
{
uint64_t size ;
uint64_t time_stamp ;
uint64_t modf_stamp ;
RsFileHash hash ;
} HashCacheInfo ;
void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; }
uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; }
void clear() { _files.clear(); }
bool empty() const { return _files.empty() ; }
private:
uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries
std::map<std::string, HashCacheInfo> _files ;
std::string _path ;
bool _changed ;
};
/******************************************************************************************
* FileIndexMonitor
*****************************************************************************************/
class FileIndexMonitor: public CacheSource, public RsTickingThread
{
public:
FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid, const std::string& config_dir);
virtual ~FileIndexMonitor();
/* external interface for filetransfer */
bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list<RsNodeGroupId>& parent_groups) const;
int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) ;
int SearchBoolExp(Expression *exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
int filterResults(std::list<FileEntry*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
/* external interface for local access to files */
bool convertSharedFilePath(std::string path, std::string &fullpath);
/* Interacting with CacheSource */
/* overloaded from CacheSource */
virtual bool loadLocalCache(const RsCacheData &data); /* called with stored data */
bool updateCache(const RsCacheData &data,const std::set<RsPeerId>& dest_peers); /* we call when we have a new cache for others */
/* the FileIndexMonitor inner workings */
//virtual void run(std::string& currentJob); /* overloaded from RsThread */
//void updateCycle(std::string& currentJob);
virtual void data_tick(); /* overloaded from RsThread */
void updateCycle();
// Interface for browsing dir hirarchy
int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ;
uint32_t getType(void*) const ;
int RequestDirDetails(const std::string& path, DirDetails &details) const ;
// set/update shared directories
virtual void setSharedDirectories(const std::list<SharedDirInfo>& dirs);
void getSharedDirectories(std::list<SharedDirInfo>& dirs);
void updateShareFlags(const SharedDirInfo& info) ;
void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed
void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files.
bool inDirectoryCheck();
/* util fns */
// from CacheSource
virtual bool cachesAvailable(const RsPeerId& pid, std::map<CacheId, RsCacheData> &ids) ;
protected:
// Sets/gets the duration period within which already hashed files are remembered.
//
void setRememberHashFilesDuration(uint32_t days) ;
uint32_t rememberHashFilesDuration() const ;
void setRememberHashFiles(bool) ;
bool rememberHashFiles() ;
// Remove any memory of formerly hashed files that are not shared anymore
void clearHashFiles() ;
void setPeriod(int insecs);
int getPeriod() const;
bool autoCheckEnabled() const ;
private:
/* the mutex should be locked before calling these 3. */
// Saves file indexs and update the cache. Returns the name of the main
// file index, which becomes the new reference file for mod times.
//
time_t locked_saveFileIndexes(bool update_cache) ;
// Finds the share flags associated with this file entry.
void locked_findShareFlagsAndParentGroups(FileEntry *fe, FileStorageFlags& shareflags, std::list<RsNodeGroupId> &parent_groups) const ;
std::string locked_findRealRoot(std::string base) const;
void hashFiles(const std::vector<DirContentToHash>& to_hash) ;
bool hashFile(std::string path, FileEntry &fi); /* To Implement */
/* data */
mutable RsMutex fiMutex;
FileIndex fi;
int updatePeriod;
std::map<std::string, SharedDirInfo> directoryMap; /* used by findRealRoot */
/* flags to kick - if we were busy or sleeping */
bool pendingDirs;
bool pendingForceCacheWrite;
/* flags to force Check, to tell if we're in check */
bool mForceCheck;
bool mInCheck;
std::list<SharedDirInfo> pendingDirList;
bool internal_setSharedDirectories();
HashCache hashCache ;
bool useHashCache ;
std::map<RsPeerId,RsCacheData> _cache_items_per_peer ; // stored the cache items to be sent to each peer.
// This file is the location of the current index file. When checking for new files, we compare the modification time
// of this file to the mod time of the files on the disk. This allows to now account for time-shift in the computer.
//
time_t reference_time ;
};
#endif

1516
libretroshare/src/dbase/findex.cc
View file

File diff suppressed because it is too large Load diff

278
libretroshare/src/dbase/findex.h
View file

@ -1,278 +0,0 @@
/*
* RetroShare FileCache Module: findex.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef FILE_INDEX_H
#define FILE_INDEX_H
#include <string>
#include <map>
#include <set>
#if __MACH__
#include <unordered_set>
#else
#include <tr1/unordered_set>
#endif
#include <list>
#include <vector>
#include <stdint.h>
#include "retroshare/rstypes.h"
/******************************************************************************************
* The Key Data Types for the File Index:
FileEntry : Information about a single file.
DirEntry : Information about a directory and its children
PersonEntry : Information the root of a FileIndex.
FileIndex : A Collection of root directories, with a set of functions to manipulate them.
In terms of retroshare, There will be a single 'Local' FileIndex used to store
the shared files, and a set of 'Remote' FileIndices which are used to store
the available files of all the peers.
******************************************************************************************/
/******************************************************************************************
STILL TODO:
(1) Load/Store a FileIndex to file...
int FileIndex::loadIndex(FILE *input);
int FileIndex::saveIndex(FILE *input);
This can be done in a recursive manner, or handled completely within FileIndex.
(2) Search Functions for Partial File Names and Hashes.
int FileIndex::searchHash(std::string hash, std::list<FileEntry> &results);
int FileIndex::searchTerms(std::list<string> terms, std::list<FileEntry> &results);
This is probably best done in a recursive manner.
The search could also be extended to handle complex Boolean searches such as :
match (size > 100K) && (name contains 'Blue') .... if anyone is interested.
But this can get quite complicated, and can be left to a later date.
******************************************************************************************/
/******************************************************************************************
* FileEntry
*****************************************************************************************/
#include <util/smallobject.h>
class DirEntry;
class FileEntry: public RsMemoryManagement::SmallObject
{
public:
FileEntry()
: size(0), modtime(0), pop(0), updtime(0), parent(NULL), row(0)
{ return; }
virtual ~FileEntry() { return; }
virtual uint32_t type() const { return DIR_TYPE_FILE ; }
virtual int print(std::string &out);
/* Data */
std::string name;
RsFileHash hash;
uint64_t size; /* file size */
time_t modtime; /* modification time - most recent mod time for a sub entry for dirs */
int pop; /* popularity rating */
time_t updtime; /* last updated */
/* References for easy manipulation */
DirEntry *parent;
int row;
std::list<std::string> parent_groups ;
};
/******************************************************************************************
* DirEntry
*****************************************************************************************/
class DirEntry: public FileEntry
{
public:
DirEntry() : most_recent_time(0) {}
/* cleanup */
virtual ~DirEntry();
/* update local entries */
DirEntry * updateDir(const FileEntry& fe, time_t updtime);
FileEntry * updateFile(const FileEntry& fe, time_t updtime);
virtual uint32_t type() const { return DIR_TYPE_DIR ; }
int checkParentPointers();
int updateChildRows();
/* remove local entries */
int removeFile(const std::string& name);
int removeDir(const std::string& name);
int removeOldDir(const std::string& name, time_t old); /* checks ts first */
/* recursive cleanup */
int removeOldEntries(time_t old, bool recursive);
/* recursive searches */
DirEntry * findOldDirectory(time_t old);
DirEntry * findDirectory(const std::string& path);
/* recursive update directory mod/pop values */
int updateDirectories(const std::string& path, int pop, int modtime);
/* output */
int print(std::string &out);
int saveEntry(std::string &out);
void writeDirInfo(std::string&);
void writeFileInfo(std::string&);
/* Data */
std::string path; /* full path (includes name) */
std::map<std::string, DirEntry *> subdirs;
std::map<std::string, FileEntry *> files;
time_t most_recent_time; /* last updated */
/* Inherited members from FileEntry:
int size - count for dirs
std::string name; - directory name
std::string hash; - not used
int size; - not used
int modtime; - most recent modication time of any child file (recursive)
int pop; - most popular child file (recursive)
int updtime; - last updated
*/
};
/******************************************************************************************
* PersonEntry
*****************************************************************************************/
class PersonEntry: public DirEntry
{
public:
/* cleanup */
PersonEntry(const RsPeerId& pid) : id(pid) { return; }
virtual ~PersonEntry() { return; }
DirEntry &operator=(DirEntry &src)
{
DirEntry *pdest = this;
(*pdest) = src;
return *this;
}
virtual uint32_t type() const { return DIR_TYPE_PERSON ; }
/* Data */
RsPeerId id;
/* Inherited members from FileEntry:
int size - count for dirs
std::string name; - directory name
std::string hash; - not used
int size; - not used
int modtime; - most recent modication time of any child file (recursive)
int pop; - most popular child file (recursive)
int updtime; - last updated
*/
};
/******************************************************************************************
* FileIndex
*****************************************************************************************/
class Expression;
class FileIndex
{
public:
FileIndex(const RsPeerId& pid);
~FileIndex();
/* control root entries */
int setRootDirectories(const std::list<std::string> &inlist, time_t utime);
int getRootDirectories(std::list<std::string> &outlist);
/* update (index building) */
DirEntry * updateDirEntry(const std::string& path, const FileEntry& fe, time_t utime);
FileEntry * updateFileEntry(const std::string& path, const FileEntry& fe, time_t utime);
DirEntry * findOldDirectory(time_t old); /* finds directories older than old */
int removeOldDirectory(const std::string& fpath, const std::string& name, time_t old);
int cleanOldEntries(time_t old); /* removes entries older than old */
/* debug */
int printFileIndex(std::string &out);
int printFileIndex(std::ostream &out);
/* load/save to file */
int loadIndex(const std::string& filename, const RsFileHash &expectedHash, uint64_t size);
int saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size, const std::set<std::string>& forbidden_roots);
/* search through this index */
int searchTerms(const std::list<std::string>& terms, std::list<FileEntry *> &results) const;
int searchHash(const RsFileHash& hash, std::list<FileEntry *> &results) const;
int searchBoolExp(Expression * exp, std::list<FileEntry *> &results) const;
/// Recursively compute the maximum modification time of children.
/// Used to easily retrieve mose recent files.
//
void updateMaxModTime() ;
void RecursUpdateMaxModTime(DirEntry *) ;
PersonEntry *root;
#ifdef __MACH__
static std::unordered_set<void*> _pointers ;
#else
static std::tr1::unordered_set<void*> _pointers ;
#endif
static void registerEntry(void*p) ;
static void unregisterEntry(void*p) ;
static bool isValid(void*p) ;
/// Fills up details from the data contained in ref.
//
static bool extractData(void *ref,DirDetails& details) ;
static uint32_t getType(void *ref) ;
void *findRef(const std::string& path) const ;
bool extractData(const std::string& path,DirDetails& details) const ;
void updateHashIndex() ;
void recursUpdateHashIndex(DirEntry *) ;
std::map<RsFileHash,FileEntry*> _file_hashes ;
};
#endif

459
libretroshare/src/dbase/fistore.cc
View file

@ -1,459 +0,0 @@
/*
* RetroShare FileCache Module: fistore.cc
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include <time.h>
#include "rsserver/p3face.h"
#include "dbase/fistore.h"
#include "retroshare/rsexpr.h"
#include "retroshare/rsfiles.h"
#include "serialiser/rsserviceids.h"
#include "pqi/p3peermgr.h"
#include "pqi/p3notify.h"
FileIndexStore::FileIndexStore(CacheStrapper *cs, CacheTransfer *cft,
p3PeerMgr *cnmgr, RsPeerId ownid, std::string cachedir)
:CacheStore(RS_SERVICE_TYPE_FILE_INDEX, false, cs, cft, cachedir),
localId(ownid), localindex(NULL), mPeerMgr(cnmgr)
{
return;
}
FileIndexStore::~FileIndexStore()
{
/* clean up the Index */
return;
}
/***
* #define FIS_DEBUG2 1
* #define FIS_DEBUG 1
**/
/* actual load, once data available */
int FileIndexStore::loadCache(const RsCacheData &data)
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() hash: " << data.hash << std::endl;
std::cerr << "FileIndexStore::loadCache() path: " << data.path << std::endl;
std::cerr << "FileIndexStore::loadCache() name: " << data.name << std::endl;
std::cerr << "FileIndexStore::loadCache() size: " << data.size << std::endl;
#endif
/* do Callback */
AboutToModify();
/* lock it up */
lockData();
FileIndex *fiold = NULL;
bool local = (data.pid == localId);
std::map<RsPeerId, FileIndex *>::iterator it;
/* remove old cache */
if (local)
{
fiold = localindex;
localindex = NULL;
}
else if (indices.end() != (it = indices.find(data.pid)))
{
fiold = it->second;
indices.erase(it);
//delete fi;
}
if(mPeerMgr->isFriend(data.pid))
{
// We discard file lists from non friends. This is the place to remove file lists of deleted friends
// from the cache. Doing this, the file list still shows in a session where we deleted a friend, but will be removed
// at next restart.
//
/* load Cache */
FileIndex *finew = new FileIndex(data.pid);
if (finew->loadIndex(data.path + '/' + data.name, data.hash, data.size))
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() Succeeded!" << std::endl;
#endif
/* This is not the place to set the peername.
* It is a hack, which makes it programmatically impossible
* to get the file data out....
*
* peername should not be used in dbase.
*/
finew->root->name = data.pid.toStdString();
if (local)
{
localindex = finew;
}
else
{
indices[data.pid] = finew;
}
delete fiold;
/* store in tale */
locked_storeCacheEntry(data);
}
else
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() Failed!" << std::endl;
#endif
/* reinstall the old one! */
delete finew;
if (fiold)
{
if (local)
{
localindex = fiold;
}
else
{
indices[data.pid] = fiold;
}
}
}
}
#ifdef FIS_DEBUG
else
std::cerr << "Discarding file list from deleted peer " << data.pid << std::endl ;
#endif
/* need to correct indices(row) for the roots of the FileIndex */
int i = 0;
for(it = indices.begin(); it != indices.end(); ++it)
{
(it->second)->root->row = i++;
it->second->FileIndex::updateMaxModTime() ;
}
if (localindex)
{
localindex->root->row = 0;
localindex->updateMaxModTime() ;
}
unlockData();
ModCompleted();
bool ret = false;
return ret;
}
/* Search Interface - For Directory Access */
int FileIndexStore::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails& details) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator it = indices.find(uid);
bool found = true;
if (it != indices.end())
found = it->second->extractData(path,details) ;
unlockData();
return found ;
#ifdef OLD_STUFF_TO_REMOVE
/* lock it up */
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator it;
it = indices.find(uid);
bool found = true;
if (it != indices.end())
{
//DirEntry *fdir = (it->second).lookupDirectory(path);
/* translate it
*/
bool b = FileIndex::extractData((it->second)->root,details) ;
found = found && b ;
}
else
found = false;
unlockData();
return found;
#endif
}
int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const
{
/* remove unused parameter warnings */
(void) flags;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl;
#endif
std::map<RsPeerId, FileIndex *>::const_iterator pit;
lockData();
// checked by FileIndex::extractData
// if(ref != NULL && !FileIndex::isValid(ref))
// {
// unlockData() ;
// return false ;
// }
/* so cast *ref to a DirEntry */
/* root case */
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() CHKS" << std::endl;
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
(pit->second)->root->checkParentPointers();
}
#endif
if (ref == NULL)
{
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() ref=NULL (root)" << std::endl;
#endif
/* get remote root entries */
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
/*
*/
FileIndex *fileIndex = pit->second;
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = fileIndex->root->name;
stub.ref = fileIndex->root;
details.children.push_back(stub);
}
details.parent = NULL;
details.prow = -1;
details.ref = NULL;
details.type = DIR_TYPE_ROOT;
details.name = "";
details.hash.clear() ;
details.path = "";
details.count = indices.size();
details.age = 0;
details.flags.clear() ;
details.min_age = 0;
unlockData();
return true ;
}
bool b = FileIndex::extractData(ref,details) ;
unlockData();
return b;
}
uint32_t FileIndexStore::getType(void *ref) const
{
lockData() ;
uint32_t b = FileIndex::getType(ref) ;
unlockData();
return b;
}
int FileIndexStore::SearchHash(const RsFileHash& hash, std::list<FileDetail> &results) const
{
lockData();
results.clear() ;
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
time_t now = time(NULL);
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash()" << std::endl;
#endif
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash() Searching: Peer ";
std::cerr << pit->first << std::endl;
#endif
firesults.clear();
(pit->second)->searchHash(hash, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
FileDetail fd;
fd.id = pit->first;
fd.name = (*rit)->name;
fd.hash = (*rit)->hash;
fd.path = ""; /* TODO */
fd.size = (*rit)->size;
fd.age = now - (*rit)->modtime;
fd.rank = (*rit)->pop;
results.push_back(fd);
}
}
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash() Found " << results.size();
std::cerr << " Results from " << indices.size() << " Peers" << std::endl;
#endif
unlockData();
return results.size();
}
int FileIndexStore::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
results.clear() ;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchKeywords()" << std::endl;
#endif
if(flags & RS_FILE_HINTS_REMOTE)
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
firesults.clear();
(pit->second)->searchTerms(keywords, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!FileIndex::extractData(*rit, dd))
continue ;
results.push_back(dd);
}
}
if(flags & RS_FILE_HINTS_LOCAL)
if (localindex)
{
firesults.clear();
localindex->searchTerms(keywords, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!FileIndex::extractData(*rit, dd))
continue ;
dd.id.clear() ;
results.push_back(dd);
}
}
unlockData();
return results.size();
}
int FileIndexStore::searchBoolExp(Expression * exp, std::list<DirDetails> &results) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::searchBoolExp()" << std::endl;
#endif
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
firesults.clear();
(pit->second)->searchBoolExp(exp, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
FileIndex::extractData(*rit, dd);
results.push_back(dd);
}
}
/* finally search local files */
if (localindex)
{
firesults.clear();
localindex->searchBoolExp(exp, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
FileIndex::extractData(*rit, dd);
dd.id.clear() ;
results.push_back(dd);
}
}
unlockData();
return results.size();
}
int FileIndexStore::AboutToModify()
{
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
return 1;
}
int FileIndexStore::ModCompleted()
{
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
return 1;
}

102
libretroshare/src/dbase/fistore.h
View file

@ -1,102 +0,0 @@
/*
* RetroShare FileCache Module: fistore.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_FILE_INDEX_STORE_H
#define MRK_FILE_INDEX_STORE_H
/**********
* Stores the FileCaches of the Peers
* must implement 'loadCache' to
*
* This class is also accessed by the GUI....
* and the FileTransfer class.
*
*/
class p3PeerMgr ;
#include "dbase/findex.h"
#include "dbase/cachestrapper.h"
#include "retroshare/rsiface.h"
class FileStoreResult
{
public:
std::string id;
std::string path;
std::string hash;
std::string name;
};
class NotifyCallback
{
public:
NotifyCallback() { return; }
virtual ~NotifyCallback() { return; }
virtual void AboutToModify() { return; }
virtual void ModCompleted() { return; }
};
class Expression;
class FileIndexStore: public CacheStore
{
public:
FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, p3PeerMgr *pmgr, RsPeerId ownid, std::string cachedir);
virtual ~FileIndexStore();
/* virtual functions overloaded by cache implementor */
virtual int loadCache(const RsCacheData &data); /* actual load, once data available */
/* Search Interface - For FileTransfer Lookup */
int SearchHash(const RsFileHash &hash, std::list<FileDetail> &results) const;
/* Search Interface - For Search Interface */
int SearchKeywords(std::list<std::string> terms, std::list<DirDetails> &results,FileSearchFlags flags) const;
/* Search Interface - for Adv Search Interface */
int searchBoolExp(Expression * exp, std::list<DirDetails> &results) const;
/* Search Interface - For Directory Access */
int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) const;
int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const;
uint32_t getType(void *ref) const ;
private:
int AboutToModify();
int ModCompleted();
std::map<RsPeerId, FileIndex *> indices;
RsPeerId localId;
FileIndex *localindex;
p3PeerMgr *mPeerMgr ;
};
#endif

289
libretroshare/src/dbase/rsexpr.cc
View file

@ -1,289 +0,0 @@
/*
* rs-core/src/dbase: rsexpr.cc
*
* RetroShare C++ Interface.
*
* Copyright 2007-2008 by Kashif Kaleem.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "dbase/findex.h"
#include "retroshare/rsexpr.h"
#include <algorithm>
#include <functional>
/******************************************************************************************
eval functions of relational expressions.
******************************************************************************************/
template<>
void RelExpression<int>::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(LowerValue) ;
e._ints.push_back(HigherValue) ;
}
bool DateExpression::eval(FileEntry *file)
{
return evalRel(file->modtime);
}
bool SizeExpressionMB::eval(FileEntry *file)
{
return evalRel((int)(file->size/(uint64_t)(1024*1024)));
}
bool SizeExpression::eval(FileEntry *file)
{
return evalRel(file->size);
}
bool PopExpression::eval(FileEntry *file)
{
return evalRel(file->pop);
}
/******************************************************************************************
Code for evaluating string expressions
******************************************************************************************/
bool NameExpression::eval(FileEntry *file)
{
return evalStr(file->name);
}
bool PathExpression::eval(FileEntry *file){
std::string path;
/*Construct the path of this file*/
DirEntry * curr = file->parent;
while ( curr != NULL ){
path = curr->name+"/"+ path;
curr = curr->parent;
}
return evalStr(path);
}
bool ExtExpression::eval(FileEntry *file){
std::string ext;
/*Get the part of the string after the last instance of . in the filename */
size_t index = file->name.find_last_of('.');
if (index != std::string::npos) {
ext = file->name.substr(index+1);
if (ext != "" ){
return evalStr(ext);
}
}
return false;
}
bool HashExpression::eval(FileEntry *file){
return evalStr(file->hash.toStdString());
}
/*Check whether two strings are 'equal' to each other*/
static bool StrEquals(const std::string & str1, const std::string & str2,
bool IgnoreCase ){
if ( str1.size() != str2.size() ){
return false;
} else if (IgnoreCase) {
std::equal( str1.begin(), str1.end(),
str2.begin(), CompareCharIC() );
}
return std::equal( str1.begin(), str1.end(),
str2.begin());
}
/*Check whether one string contains the other*/
static bool StrContains( const std::string & str1, const std::string & str2,
bool IgnoreCase){
std::string::const_iterator iter ;
if (IgnoreCase) {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end(), CompareCharIC() );
} else {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end());
}
return ( iter != str1.end() );
}
bool StringExpression :: evalStr ( const std::string &str ){
std::list<std::string>::iterator iter;
switch (Op) {
case ContainsAllStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str, *iter, IgnoreCase) == false ){
return false;
}
}
return true;
break;
case ContainsAnyStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
case EqualsString:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrEquals (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
default:
return false;
}
return false;
}
/*************************************************************************
* linearization code
*************************************************************************/
void CompoundExpression::linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_COMP) ;
e._ints.push_back(Op) ;
Lexp->linearize(e) ;
Rexp->linearize(e) ;
}
void StringExpression::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(IgnoreCase) ;
e._ints.push_back(terms.size()) ;
for(std::list<std::string>::const_iterator it(terms.begin());it!=terms.end();++it)
e._strings.push_back(*it) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e)
{
int i=0,j=0,k=0 ;
return toExpr(e,i,j,k) ;
}
void LinearizedExpression::readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list<std::string>& strings,bool& b,StringOperator& op)
{
op = static_cast<StringOperator>(e._ints[n_ints++]) ;
b = e._ints[n_ints++] ;
int n = e._ints[n_ints++] ;
strings.clear() ;
for(int i=0;i<n;++i)
strings.push_back(e._strings[n_strings++]) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e,int& n_tok,int& n_ints,int& n_strings)
{
LinearizedExpression::token tok = static_cast<LinearizedExpression::token>(e._tokens[n_tok++]) ;
switch(tok)
{
case EXPR_DATE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new DateExpression(op,lv,hv) ;
}
case EXPR_POP: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new PopExpression(op,lv,hv) ;
}
case EXPR_SIZE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpression(op,lv,hv) ;
}
case EXPR_HASH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new HashExpression(op,strings) ;
}
case EXPR_NAME: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new NameExpression(op,strings,b) ;
}
case EXPR_PATH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_EXT: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_COMP: {
LogicalOperator op = static_cast<LogicalOperator>(e._ints[n_ints++]) ;
Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ;
Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ;
return new CompoundExpression(op,e1,e2) ;
}
case EXPR_SIZE_MB: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpressionMB(op,lv,hv) ;
}
default:
std::cerr << "No expression match the current value " << tok << std::endl ;
return NULL ;
}
}

132
libretroshare/src/file_sharing/README.txt
View file

@ -40,9 +40,35 @@ Big picture
- the same file should be able to be held by two different directories. Hash search will return a single hit.
- the previously existing PersonEntry had no field and was overloading DirEntry, with overwritten file names, hashes etc. Super bad!
Directory storage file format
-----------------------------
* should be extensible (xml or binary format? Binary, because it's going to be encrypted anyway)
=> works with binary fields
=>
[= version =]
[= peer id =]
[= num entries =]
[= some information =]
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
...
2 1-5 v 2 1-5 v
* entry content
Tag | Content | Size
----------------+--------------------------------------+------
01 | sha1 hash | 20
01 | sha1^2 hash | 20
02 | file name | < 512
03 | file size | 8
04 | dir name | < 512
05 | last modif time local | 4
06 | last modif time including sub-dirs | 4
Classes
-------
Rs
p3ShareManager
- tick()
@ -75,6 +101,29 @@ Classes
- parent groups
- group flags
Best data structure for file index
----------------------------------
| Hash map map list
----------------+-----------------+------------+--------------
Adding | Constant | log(n) | O(n)
Hash search | Constant | log(n) | O(n)
Name/exp search | O(n) | O(n) | O(n)
Recursive browse| Constant | log(n) | O(n)
Should we use the same struct for files and directories?
Sol 1:
DirClass + PersonClass + FileEntry class
- each has pointers to elements list of the same type
- lists are handled for Files (all file entries),
Directories are represented by the hash of the full path
Sol 2:
Same class for all elements, in a single hash map. Each element is
defined by its type (Dir, Person, File) which all have a hash.
Syncing between peers
---------------------
@ -83,19 +132,84 @@ Generating sync events
- for each directory, in breadth first order
- if directory has changed, or last update is old
=> push a sync request
- store the peer's last up time. Compare with peer uptimes recursively.
* Server side
- after a change, broadcast a "directory changed" packet to all connected friends
* directoy updater
- crawl through directories
- compare TS of files, missing files, new files
- feed a queue of files to hash
- directory whatcher gets notified when files are hashed
- a separate component hashes files (FileHashingProcess)
DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy)
| |
| |
| |
+-----------------------+------------------+
| |
Shared File Service |
| |
| |
+----------- own file list -------+---------- Encrypted/compressed save to disk
| | |
+----------- friend file lists ---+
Roadmap
-------
- complete this file until a proper description of the whole thing is achieved.
- create a new directory and implement the .h for the basic functionality
- look into existing code in ftServer for the integration, but don't change anything yet
- setup class hierarchy
- merge hash cache into file lists.
[X] complete this file until a proper description of the whole thing is achieved.
[X] create a new directory and implement the .h for the basic functionality
[ ] look into existing code in ftServer for the integration, but don't change anything yet
[X] setup class hierarchy
[ ] merge hash cache into file lists.
[ ] new format for saving of FileIndex to make it locally encrypted, compact and extensible
[ ] create basic directory functionality with own files: re-hash, and store
[ ] display own files in GUI, with proper update and working sort
TODO
====
[ ] directory handler
[ ] abstract functions to keep a directory and get updates to it.
[ ] hierarchical storage representation.
[ ] allow add/delete entries
[ ] auto-cleanup
[ ] directory updater
[ ] abstract layer
[ ] crawls the directory and ask updates
[ ] derive local directory updater
[ ] crawl local files, and asks updates to storage class
[ ] derive remote directory updater
[ ] crawl stored files, and request updates to storage class
[ ] load/save of directory content. Should be extensible
[ ] p3FileLists with minimal functonality: no exchange. Only storage of own file lists
[ ] service (items) for p3FileLists
[ ] connect RemoteDirModel to new system
[ ] test GUI functions
[ ] test update between peers
- optionally
- change the saving system of FileIndex to make it locally encrypted and compact

1099
libretroshare/src/file_sharing/dir_hierarchy.cc Normal file
View file

File diff suppressed because it is too large Load diff

183
libretroshare/src/file_sharing/dir_hierarchy.h Normal file
View file

@ -0,0 +1,183 @@
/*
* RetroShare C++ Internal directory hierarchy class.
*
* file_sharing/dir_hierarchy.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#include "directory_storage.h"
class InternalFileHierarchyStorage
{
public:
class FileStorageNode
{
public:
static const uint32_t TYPE_UNKNOWN = 0x0000 ;
static const uint32_t TYPE_FILE = 0x0001 ;
static const uint32_t TYPE_DIR = 0x0002 ;
virtual ~FileStorageNode() {}
virtual uint32_t type() const =0;
DirectoryStorage::EntryIndex parent_index;
uint32_t row ;
};
class FileEntry: public FileStorageNode
{
public:
FileEntry() : file_size(0), file_modtime(0) {}
FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {}
FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {}
virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
virtual ~FileEntry() {}
// local stuff
std::string file_name ;
uint64_t file_size ;
time_t file_modtime;
RsFileHash file_hash ;
};
class DirEntry: public FileStorageNode
{
public:
DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),dir_most_recent_time(0),dir_update_time(0) {}
virtual ~DirEntry() {}
virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
// local stuff
std::string dir_name ;
std::string dir_parent_path ;
RsFileHash dir_hash ;
std::vector<DirectoryStorage::EntryIndex> subdirs ;
std::vector<DirectoryStorage::EntryIndex> subfiles ;
time_t dir_modtime;
time_t dir_most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below.
time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info.
};
// class stuff
InternalFileHierarchyStorage() ;
bool load(const std::string& fname) ;
bool save(const std::string& fname) ;
int parentRow(DirectoryStorage::EntryIndex e);
bool isIndexValid(DirectoryStorage::EntryIndex e) const;
bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const;
bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs);
bool removeDirectory(DirectoryStorage::EntryIndex indx) ;
bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const;
bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files);
bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash);
bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time);
bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector<RsFileHash> &subdirs_hash, const std::vector<FileEntry> &subfiles_array);
// TS get/set functions. Take one of the class members as argument.
bool getTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) const;
bool setTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) ;
// Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method.
time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index);
// hash stuff
bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ;
bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ;
bool getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ;
// file/dir access and modification
bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e
uint32_t mRoot ;
std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to
// all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that
// are not referenced.
friend class DirectoryStorage ; // only class that can use this.
friend class LocalDirectoryStorage ; // only class that can use this.
// Low level stuff. Should normally not be used externally.
const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const;
const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const;
const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const;
uint32_t getType(DirectoryStorage::EntryIndex indx) const;
DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index);
DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index);
// search. SearchHash is logarithmic. The other two are linear.
bool searchHash(const RsFileHash& hash,std::list<DirectoryStorage::EntryIndex>& results);
int searchBoolExp(RsRegularExpression::Expression * exp, std::list<DirectoryStorage::EntryIndex> &results) const ;
int searchTerms(const std::list<std::string>& terms, std::list<DirectoryStorage::EntryIndex> &results) const ;
bool check(std::string& error_string) const ;// checks consistency of storage.
void print() const;
private:
void recursPrint(int depth,DirectoryStorage::EntryIndex node) const;
static bool nodeAccessError(const std::string& s);
static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ;
// Allocates a new entry in mNodes, possible re-using an empty slot and returns its index.
DirectoryStorage::EntryIndex allocateNewIndex();
// Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
// phase. That allows to keep file information when moving them around.
bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir);
// Map of the hash of all files. The file hashes are the sha1sum of the file data.
// is used for fast search access for FT.
// Note: We should try something faster than std::map. hash_map??
// Unlike directories, multiple files may have the same hash. So this cannot be used for anything else than FT.
std::map<RsFileHash,DirectoryStorage::EntryIndex> mFileHashes ;
// The directory hashes are the sha1sum of the
// full public path to the directory.
// The later is used by synchronisation items in order
// to avoid sending explicit EntryIndex values.
// This is kept separate from mFileHashes because the two are used
// in very different ways.
//
std::map<RsFileHash,DirectoryStorage::EntryIndex> mDirHashes ;
};

13
libretroshare/src/file_sharing/directory_list.h Normal file
View file

@ -0,0 +1,13 @@
// This class keeps a shared directory. It's quite the equivalent of the old "FileIndex" class
// The main difference is that it is
// - extensible
// - fast to search (at least for hashes). Should provide possibly multiple search handles for
// the same file, e.g. if connexion is encrypted.
// - abstracts the browsing in a same manner.
//
class SharedDirectoryList
{
public:
DirEntry mRoot ;
};

806
libretroshare/src/file_sharing/directory_storage.cc Normal file
View file

@ -0,0 +1,806 @@
/*
* RetroShare File list storage system.
*
* file_sharing/directory_storage.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include <set>
#include <time.h>
#include "serialiser/rstlvbinary.h"
#include "retroshare/rspeers.h"
#include "util/rsdir.h"
#include "util/rsstring.h"
#include "file_sharing_defaults.h"
#include "directory_storage.h"
#include "dir_hierarchy.h"
#include "filelist_io.h"
//#define DEBUG_REMOTE_DIRECTORY_STORAGE 1
/******************************************************************************************************************/
/* Iterators */
/******************************************************************************************************************/
DirectoryStorage::DirIterator::DirIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
{
mStorage = s->mFileHierarchy ;
mParentIndex = i;
mDirTabIndex = 0;
}
DirectoryStorage::FileIterator::FileIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
{
mStorage = s->mFileHierarchy ;
mParentIndex = i;
mFileTabIndex = 0;
}
DirectoryStorage::DirIterator& DirectoryStorage::DirIterator::operator++()
{
++mDirTabIndex ;
return *this;
}
DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
{
++mFileTabIndex ;
return *this;
}
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; }
DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
RsFileHash DirectoryStorage::FileIterator::hash() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_hash):RsFileHash(); }
uint64_t DirectoryStorage::FileIterator::size() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_size):0; }
std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); }
time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; }
std::string DirectoryStorage::DirIterator::name() const { const InternalFileHierarchyStorage::DirEntry *d = mStorage->getDirEntry(**this) ; return d?(d->dir_name):std::string(); }
/******************************************************************************************************************/
/* Directory Storage */
/******************************************************************************************************************/
DirectoryStorage::DirectoryStorage(const RsPeerId &pid)
: mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString())
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy = new InternalFileHierarchyStorage();
}
DirectoryStorage::EntryIndex DirectoryStorage::root() const
{
return EntryIndex(0) ;
}
int DirectoryStorage::parentRow(EntryIndex e) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->parentRow(e) ;
}
bool DirectoryStorage::getChildIndex(EntryIndex e,int row,EntryIndex& c) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->getChildIndex(e,row,c) ;
}
uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
switch(mFileHierarchy->getType(indx))
{
case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR: return DIR_TYPE_DIR ;
case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE: return DIR_TYPE_FILE ;
default:
return DIR_TYPE_UNKNOWN;
}
}
bool DirectoryStorage::getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); }
bool DirectoryStorage::getDirectoryRecursModTime(EntryIndex index,time_t& rec_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); }
bool DirectoryStorage::getDirectoryLocalModTime (EntryIndex index,time_t& loc_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); }
bool DirectoryStorage::setDirectoryUpdateTime (EntryIndex index,time_t update_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); }
bool DirectoryStorage::setDirectoryRecursModTime(EntryIndex index,time_t rec_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); }
bool DirectoryStorage::setDirectoryLocalModTime (EntryIndex index,time_t loc_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); }
bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map<std::string,time_t>& subdirs)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ;
locked_check() ;
return res ;
}
bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
locked_check() ;
return res ;
}
bool DirectoryStorage::removeDirectory(const EntryIndex& indx)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->removeDirectory(indx);
locked_check();
return res ;
}
void DirectoryStorage::locked_check()
{
std::string error ;
if(!mFileHierarchy->check(error))
std::cerr << "Check error: " << error << std::endl;
}
bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->updateFile(index,hash,fname,size,modf_time);
}
bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->updateHash(index,hash);
}
int DirectoryStorage::searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchHash(hash,results);
}
void DirectoryStorage::load(const std::string& local_file_name)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->load(local_file_name);
}
void DirectoryStorage::save(const std::string& local_file_name)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->save(local_file_name);
}
void DirectoryStorage::print()
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->print();
}
int DirectoryStorage::searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchTerms(terms,results);
}
int DirectoryStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list<EntryIndex> &results) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchBoolExp(exp,results);
}
bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
d.children.clear() ;
time_t now = time(NULL) ;
uint32_t type = mFileHierarchy->getType(indx) ;
d.ref = (void*)(intptr_t)indx ;
if (type == InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR) /* has children --- fill */
{
const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ;
/* extract all the entries */
for(DirectoryStorage::DirIterator it(this,indx);it;++it)
{
DirStub stub;
stub.type = DIR_TYPE_DIR;
stub.name = it.name();
stub.ref = (void*)(intptr_t)*it; // this is updated by the caller, who knows which friend we're dealing with
d.children.push_back(stub);
}
for(DirectoryStorage::FileIterator it(this,indx);it;++it)
{
DirStub stub;
stub.type = DIR_TYPE_FILE;
stub.name = it.name();
stub.ref = (void*)(intptr_t)*it;
d.children.push_back(stub);
}
d.type = DIR_TYPE_DIR;
d.hash.clear() ;
d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size();
d.min_age = now - dir_entry->dir_most_recent_time ;
d.name = dir_entry->dir_name;
d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ;
d.parent = (void*)(intptr_t)dir_entry->parent_index ;
if(indx == 0)
{
d.type = DIR_TYPE_PERSON ;
d.name = mPeerId.toStdString();
}
}
else if(type == InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)
{
const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ;
d.type = DIR_TYPE_FILE;
d.count = file_entry->file_size;
d.min_age = now - file_entry->file_modtime ;
d.name = file_entry->file_name;
d.hash = file_entry->file_hash;
d.age = now - file_entry->file_modtime;
d.parent = (void*)(intptr_t)file_entry->parent_index ;
const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index);
if(parent_dir_entry != NULL)
d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ;
else
d.path = "" ;
}
else
return false;
d.flags.clear() ;
return true;
}
bool DirectoryStorage::getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->getDirHashFromIndex(index,hash) ;
}
bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->getIndexFromDirHash(hash,index) ;
}
/******************************************************************************************************************/
/* Local Directory Storage */
/******************************************************************************************************************/
void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo>& lst)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
// Chose virtual name if not supplied, and remove duplicates.
std::set<std::string> virtual_names ; // maps virtual to real name
std::list<SharedDirInfo> processed_list ;
for(std::list<SharedDirInfo>::const_iterator it(lst.begin());it!= lst.end();++it)
{
int i=0;
std::string candidate_virtual_name = it->virtualname ;
if(candidate_virtual_name.empty())
candidate_virtual_name = RsDirUtil::getTopDir(it->filename);
while(virtual_names.find(candidate_virtual_name) != virtual_names.end())
rs_sprintf_append(candidate_virtual_name, "-%d", ++i);
SharedDirInfo d(*it);
d.virtualname = candidate_virtual_name ;
processed_list.push_back(d) ;
virtual_names.insert(candidate_virtual_name) ;
}
mLocalDirs.clear();
for(std::list<SharedDirInfo>::const_iterator it(processed_list.begin());it!=processed_list.end();++it)
mLocalDirs[it->filename] = *it;
mTSChanged = true ;
}
void LocalDirectoryStorage::getSharedDirectoryList(std::list<SharedDirInfo>& lst)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
lst.clear();
for(std::map<std::string,SharedDirInfo>::iterator it(mLocalDirs.begin());it!=mLocalDirs.end();++it)
lst.push_back(it->second) ;
}
static bool sameLists(const std::list<RsNodeGroupId>& l1,const std::list<RsNodeGroupId>& l2)
{
std::list<RsNodeGroupId>::const_iterator it1(l1.begin()) ;
std::list<RsNodeGroupId>::const_iterator it2(l2.begin()) ;
for(; (it1!=l1.end() && it2!=l2.end());++it1,++it2)
if(*it1 != *it2)
return false ;
return it1 == l1.end() && it2 == l2.end() ;
}
void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info)
{
bool changed = false ;
{
RS_STACK_MUTEX(mDirStorageMtx) ;
std::map<std::string,SharedDirInfo>::iterator it = mLocalDirs.find(info.filename) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl;
return ;
}
// we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just
// want to make sure that if they are different, flags get updated.
if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname)
{
it->second = info;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl;
#endif
changed = true ;
}
}
if(changed)
{
setDirectoryLocalModTime(0,time(NULL)) ;
mTSChanged = true ;
}
}
bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath)
{
std::string shpath = RsDirUtil::removeRootDir(path);
std::string basedir = RsDirUtil::getRootDir(path);
std::string realroot ;
{
RS_STACK_MUTEX(mDirStorageMtx) ;
realroot = locked_findRealRootFromVirtualFilename(basedir);
}
if (realroot.empty())
return false;
/* construct full name */
fullpath = realroot + "/";
fullpath += shpath;
return true;
}
void LocalDirectoryStorage::notifyTSChanged()
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mTSChanged = true ;
}
void LocalDirectoryStorage::updateTimeStamps()
{
RS_STACK_MUTEX(mDirStorageMtx) ;
if(mTSChanged)
{
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Updating recursive TS for local shared dirs..." << std::endl;
#endif
time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ;
mTSChanged = false ;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl;
#endif
}
}
std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const
{
/**** MUST ALREADY BE LOCKED ****/
std::map<std::string, SharedDirInfo>::const_iterator cit = mLocalDirs.find(virtual_rootdir) ;
if (cit == mLocalDirs.end())
{
std::cerr << "(EE) locked_findRealRootFromVirtualFilename() Invalid RootDir: " << virtual_rootdir << std::endl;
return std::string();
}
return cit->second.filename;
}
bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
{
bool res = DirectoryStorage::extractData(indx,d) ;
if(!res)
return false;
// here we should update the file sharing flags
return getFileSharingPermissions(indx,d.flags,d.parent_groups) ;
}
bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info)
{
DirDetails d;
extractData(i,d) ;
if(d.type != DIR_TYPE_FILE)
{
std::cerr << "(EE) LocalDirectoryStorage: asked for file info for index " << i << " which is not a file." << std::endl;
return false;
}
info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file.
info.parent_groups = d.parent_groups;
info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_*
info.path = d.path + "/" + d.name;
info.fname = d.name;
info.hash = d.hash;
info.size = d.count;
// all this stuff below is not useful in this case.
info.mId = 0; /* (GUI) Model Id -> unique number */
info.ext.clear();
info.avail = 0; /* how much we have */
info.rank = 0;
info.age = 0;
info.queue_position =0;
info.searchId = 0; /* 0 if none */
/* Transfer Stuff */
info.transfered = 0;
info.tfRate = 0; /* in kbytes */
info.downloadStatus = FT_STATE_COMPLETE ;
std::list<TransferInfo> peers;
info.priority = SPEED_NORMAL;
info.lastTS = 0;
return true;
}
bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list<RsNodeGroupId>& parent_groups)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return locked_getFileSharingPermissions(indx,flags,parent_groups) ;
}
bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list<RsNodeGroupId> &parent_groups)
{
flags.clear() ;
parent_groups.clear();
std::string base_dir;
const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ;
if(n == NULL)
return false ;
for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;)
{
const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ;
if(e == NULL)
break ;
if(e->parent_index == 0)
{
base_dir = e->dir_name ;
break ;
}
i = e->parent_index ;
}
if(!base_dir.empty())
{
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(base_dir) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
return false ;
}
flags = it->second.shareflags;
parent_groups = it->second.parent_groups;
}
return true;
}
std::string LocalDirectoryStorage::locked_getVirtualDirName(EntryIndex indx) const
{
if(indx == 0)
return std::string() ;
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
if(dir->parent_index != 0)
return dir->dir_name ;
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(dir->dir_name) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl;
return std::string() ;
}
return it->second.virtualname ;
}
std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const
{
if(indx == 0)
return std::string() ;
std::string res ;
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
while(dir->parent_index != 0)
{
dir = mFileHierarchy->getDirEntry(dir->parent_index) ;
res += dir->dir_name + "/"+ res ;
}
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(dir->dir_name) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl;
return std::string() ;
}
return it->second.virtualname + "/" + res;
}
bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
if(dir == NULL)
{
std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl;
return false;
}
// compute list of allowed subdirs
std::vector<RsFileHash> allowed_subdirs ;
FileStorageFlags node_flags ;
std::list<RsNodeGroupId> node_groups ;
// for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter
for(uint32_t i=0;i<dir->subdirs.size();++i)
if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
{
RsFileHash hash ;
if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
{
std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ;
return false;
}
allowed_subdirs.push_back(hash) ;
}
unsigned char *section_data = NULL;
uint32_t section_size = 0;
uint32_t section_offset = 0;
// we need to send:
// - the name of the directory, its TS
// - the index entry for each subdir (the updte TS are exchanged at a higher level)
// - the file info for each subfile
//
std::string virtual_dir_name = locked_getVirtualDirName(indx) ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,virtual_dir_name )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ;
// serialise number of subdirs and number of subfiles
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ;
// serialise subdirs entry indexes
for(uint32_t i=0;i<allowed_subdirs.size();++i)
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,allowed_subdirs[i] )) return false ;
// serialise directory subfiles, with info for each of them
for(uint32_t i=0;i<dir->subfiles.size();++i)
{
unsigned char *file_section_data = NULL ;
uint32_t file_section_offset = 0 ;
uint32_t file_section_size = 0;
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
if(file == NULL)
{
std::cerr << "(EE) cannot reach file entry " << dir->subfiles[i] << " to get/send file info." << std::endl;
continue ;
}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)file->file_modtime)) return false ;
// now write the whole string into a single section in the file
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) return false ;
free(file_section_data) ;
}
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl;
#endif
bindata.bin_data = section_data ;
bindata.bin_len = section_offset ;
return true ;
}
/******************************************************************************************************************/
/* Remote Directory Storage */
/******************************************************************************************************************/
RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname)
: DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname)
{
load(fname) ;
std::cerr << "Loaded remote directory for peer " << pid << std::endl;
#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE
mFileHierarchy->print();
#endif
}
void RemoteDirectoryStorage::checkSave()
{
time_t now = time(NULL);
if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now)
{
save(mFileName);
mLastSavedTime = now ;
}
}
bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata)
{
const unsigned char *section_data = (unsigned char*)bindata.bin_data ;
uint32_t section_size = bindata.bin_len ;
uint32_t section_offset=0 ;
#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE
std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl;
#endif
std::string dir_name ;
uint32_t most_recent_time ,dir_modtime ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir_name )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) return false ;
std::cerr << " dir name : \"" << dir_name << "\"" << std::endl;
std::cerr << " most recent time : " << most_recent_time << std::endl;
std::cerr << " modification time : " << dir_modtime << std::endl;
// serialise number of subdirs and number of subfiles
uint32_t n_subdirs,n_subfiles ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles )) return false ;
std::cerr << " number of subdirs : " << n_subdirs << std::endl;
std::cerr << " number of files : " << n_subfiles << std::endl;
// serialise subdirs entry indexes
std::vector<RsFileHash> subdirs_hashes ;
RsFileHash subdir_hash ;
for(uint32_t i=0;i<n_subdirs;++i)
{
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,subdir_hash)) return false ;
subdirs_hashes.push_back(subdir_hash) ;
}
// deserialise directory subfiles, with info for each of them
std::vector<InternalFileHierarchyStorage::FileEntry> subfiles_array ;
for(uint32_t i=0;i<n_subfiles;++i)
{
// Read the full data section for the file
unsigned char *file_section_data = NULL ;
uint32_t file_section_size = 0;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_size)) return false ;
uint32_t file_section_offset = 0 ;
InternalFileHierarchyStorage::FileEntry f;
uint32_t modtime =0;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,f.file_name )) return false ;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,f.file_size )) return false ;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,f.file_hash )) return false ;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,modtime )) return false ;
f.file_modtime = modtime ;
free(file_section_data) ;
subfiles_array.push_back(f) ;
}
RS_STACK_MUTEX(mDirStorageMtx) ;
std::cerr << " updating dir entry..." << std::endl;
// First create the entries for each subdir and each subfile, if needed.
if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_array))
{
std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl;
return false ;
}
mChanged = true ;
return true ;
}

285
libretroshare/src/file_sharing/directory_storage.h Normal file
View file

@ -0,0 +1,285 @@
/*
* RetroShare C++ Directory Storage system.
*
* file_sharing/directory_storage.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <string>
#include <stdint.h>
#include <list>
#include "retroshare/rsids.h"
#include "retroshare/rsfiles.h"
#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
class RsTlvBinaryData ;
class InternalFileHierarchyStorage ;
class RsTlvBinaryData ;
class DirectoryStorage
{
public:
DirectoryStorage(const RsPeerId& pid) ;
virtual ~DirectoryStorage() {}
typedef uint32_t EntryIndex ;
static const EntryIndex NO_INDEX = 0xffffffff;
void save() const ;
// These functions are to be used by file transfer and file search.
virtual int searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const ;
virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list<EntryIndex> &results) const ;
virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const ;
// gets/sets the various time stamps:
//
bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; // last modification time, computed recursively over all subfiles and directories
bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; // last modification time for that index only
bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; // last time the entry was updated. This is only used on the RemoteDirectoryStorage side.
bool setDirectoryRecursModTime(EntryIndex index,time_t recurs_max_modf_TS) ;
bool setDirectoryLocalModTime (EntryIndex index,time_t modtime_TS) ;
bool setDirectoryUpdateTime (EntryIndex index,time_t update_TS) ;
uint32_t getEntryType(const EntryIndex& indx) ; // WARNING: returns DIR_TYPE_*, not the internal directory storage stuff.
virtual bool extractData(const EntryIndex& indx,DirDetails& d);
// This class allows to abstractly browse the stored directory hierarchy in a depth-first manner.
// It gives access to sub-files and sub-directories below. When using it, the client should make sure
// that the DirectoryStorage is properly locked, since the iterator cannot lock it.
//
class DirIterator
{
public:
DirIterator(const DirIterator& d) ;
DirIterator(DirectoryStorage *d,EntryIndex i) ;
DirIterator& operator++() ;
EntryIndex operator*() const ;
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
// info about the directory that is pointed by the iterator
std::string name() const ;
time_t last_modif_time() const ; // last time a file in this directory or in the directories below has been modified.
time_t last_update_time() const ; // last time this directory was updated
private:
EntryIndex mParentIndex ; // index of the parent dir.
uint32_t mDirTabIndex ; // index in the vector of subdirs.
InternalFileHierarchyStorage *mStorage ;
friend class DirectoryStorage ;
};
class FileIterator
{
public:
FileIterator(DirIterator& d); // crawls all files in specified directory
FileIterator(DirectoryStorage *d,EntryIndex e); // crawls all files in specified directory
FileIterator& operator++() ;
EntryIndex operator*() const ; // current file entry
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
// info about the file that is pointed by the iterator
std::string name() const ;
uint64_t size() const ;
RsFileHash hash() const ;
time_t modtime() const ;
private:
EntryIndex mParentIndex ; // index of the parent dir.
uint32_t mFileTabIndex ; // index in the vector of subdirs.
InternalFileHierarchyStorage *mStorage ;
};
struct FileTS
{
uint64_t size ;
time_t modtime;
};
EntryIndex root() const ; // returns the index of the root directory entry. This is generally 0.
const RsPeerId& peerId() const { return mPeerId ; } // peer ID of who owns that file list.
int parentRow(EntryIndex e) const ; // position of the current node, in the array of children at its parent node. Used by GUI for display.
bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; // returns the index of the children node at position "row" in the children nodes. Used by GUI for display.
// Sets the subdirectory/subfiles list of entry indx the supplied one, possible adding and removing directories (resp.files). New directories are set empty with
// just a name and need to be updated later on. New files are returned in a list so that they can be sent to hash cache.
//
bool updateSubDirectoryList(const EntryIndex& indx, const std::map<std::string, time_t> &subdirs) ;
bool updateSubFilesList(const EntryIndex& indx, const std::map<std::string, FileTS> &subfiles, std::map<std::string, FileTS> &new_files) ;
bool removeDirectory(const EntryIndex& indx) ;
// Updates relevant information for the file at the given index.
bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ;
bool updateHash(const EntryIndex& index,const RsFileHash& hash);
// Returns the hash of the directory at the given index and reverse. This hash is set as random the first time it is used (when updating directories). It will be
// used by the sync system to designate the directory without referring to index (index could be used to figure out the existance of hidden directories)
bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; // constant cost
bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; // log cost.
void print();
void cleanup();
protected:
void load(const std::string& local_file_name) ;
void save(const std::string& local_file_name) ;
private:
// debug
void locked_check();
// storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector.
RsPeerId mPeerId;
protected:
mutable RsMutex mDirStorageMtx ;
InternalFileHierarchyStorage *mFileHierarchy ;
};
class RemoteDirectoryStorage: public DirectoryStorage
{
public:
RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) ;
virtual ~RemoteDirectoryStorage() {}
/*!
* \brief deserialiseDirEntry
* Loads a serialised directory content coming from a friend. The directory entry needs to exist already,
* as it is created when updating the parent.
*
* \param indx index of the directory to update
* \param bindata binary data to deserialise from
* \return false when the directory cannot be found.
*/
bool deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ;
/*!
* \brief checkSave
* Checks the time of last saving, last modification time, and saves if needed.
*/
void checkSave() ;
private:
time_t mLastSavedTime ;
bool mChanged ;
std::string mFileName;
};
class LocalDirectoryStorage: public DirectoryStorage
{
public:
LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {}
virtual ~LocalDirectoryStorage() {}
/*!
* \brief [gs]etSharedDirectoryList
* Gets/sets the list of shared directories. Each directory is supplied with a virtual name (the name the friends will see), and sharing flags/groups.
* \param lst
*/
void setSharedDirectoryList(const std::list<SharedDirInfo>& lst) ;
void getSharedDirectoryList(std::list<SharedDirInfo>& lst) ;
void updateShareFlags(const SharedDirInfo& info) ;
bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ;
/*!
* \brief updateTimeStamps
* Checks recursive TS and update the if needed.
*/
void updateTimeStamps();
/*!
* \brief notifyTSChanged
* Use this to force an update of the recursive TS, when calling updateTimeStamps();
*/
void notifyTSChanged();
/*!
* \brief getFileInfo Converts an index info a full file info structure.
* \param i index in the directory structure
* \param info structure to be filled in
* \return false if the file does not exist, or is a directory,...
*/
bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ;
/*!
* \brief getFileSharingPermissions
* Computes the flags and parent groups for any index.
* \param indx index of the entry to compute the flags for
* \param flags computed flags
* \param parent_groups computed parent groups
* \return
* false if the index is not valid
* false otherwise
*/
bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list<RsNodeGroupId> &parent_groups);
virtual bool extractData(const EntryIndex& indx,DirDetails& d) ;
/*!
* \brief serialiseDirEntry
* Produced a serialised directory content listing suitable for export to friends.
*
* \param indx index of the directory to serialise
* \param bindata binary data created by serialisation
* \param client_id Peer id to be serialised to. Depending on permissions, some subdirs can be removed.
* \return false when the directory cannot be found.
*/
bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ;
private:
std::string locked_getVirtualPath(EntryIndex indx) const ;
std::string locked_getVirtualDirName(EntryIndex indx) const ;
bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list<RsNodeGroupId>& parent_groups);
std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const;
std::map<std::string,SharedDirInfo> mLocalDirs ; // map is better for search. it->first=it->second.filename
std::string mFileName;
bool mTSChanged ;
};

230
libretroshare/src/file_sharing/directory_updater.cc Normal file
View file

@ -0,0 +1,230 @@
/*
* RetroShare Directory watching system.
*
* file_sharing/directory_updater.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "util/folderiterator.h"
#include "rsserver/p3face.h"
#include "directory_storage.h"
#include "directory_updater.h"
#include "file_sharing_defaults.h"
//#define DEBUG_LOCAL_DIR_UPDATER 1
//=============================================================================================================//
// Local Directory Updater //
//=============================================================================================================//
LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds)
: mHashCache(hc),mSharedDirectories(lds)
{
mLastSweepTime = 0;
mLastTSUpdateTime = 0;
mDelayBetweenDirectoryUpdates = DELAY_BETWEEN_DIRECTORY_UPDATES;
mIsEnabled = false ;
}
bool LocalDirectoryUpdater::isEnabled() const
{
return mIsEnabled ;
}
void LocalDirectoryUpdater::setEnabled(bool b)
{
if(mIsEnabled == b)
return ;
if(b)
start() ;
else
shutdown();
mIsEnabled = b ;
}
void LocalDirectoryUpdater::data_tick()
{
time_t now = time(NULL) ;
if(now > mDelayBetweenDirectoryUpdates + mLastSweepTime)
{
sweepSharedDirectories() ;
mLastSweepTime = now;
mSharedDirectories->notifyTSChanged() ;
}
if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime)
{
mSharedDirectories->updateTimeStamps() ;
mLastTSUpdateTime = now ;
}
usleep(10*1000*1000);
}
void LocalDirectoryUpdater::forceUpdate()
{
mLastSweepTime = 0;
}
void LocalDirectoryUpdater::sweepSharedDirectories()
{
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
#endif
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure.
// - the information that is costly to compute (the hash) is store externally into a separate structure.
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
std::list<SharedDirInfo> shared_directory_list ;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
std::map<std::string,time_t> sub_dir_list ;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
sub_dir_list[(*real_dir_it).filename] = 0 ;
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ;
// now for each of them, go recursively and match both files and dirs
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
// are not necessarily in the same order.
}
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
}
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl;
#endif
// make sure list of subdirs is the same
// make sure list of subfiles is the same
// request all hashes to the hashcache
librs::util::FolderIterator dirIt(cumulated_path);
// collect subdirs and subfiles
std::map<std::string,DirectoryStorage::FileTS> subfiles ;
std::map<std::string,time_t> subdirs ;
for(;dirIt.isValid();dirIt.next())
{
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
}
// update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs) ;
std::map<std::string,DirectoryStorage::FileTS> new_files ;
mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
// now go through list of subfiles and request the hash to hashcache
for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
{
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
RsFileHash hash ;
if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash)
mSharedDirectories->updateHash(*dit,hash);
}
// go through the list of sub-dirs and recursively update
DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ;
for(std::map<std::string,time_t>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ;
}
}
bool LocalDirectoryUpdater::inDirectoryCheck() const
{
return mHashCache->isRunning();
}
void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)
{
if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash))
std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl;
mSharedDirectories->notifyTSChanged() ;
}
bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param)
{
return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ;
}
void LocalDirectoryUpdater::setFileWatchPeriod(int seconds)
{
mDelayBetweenDirectoryUpdates = seconds ;
}
uint32_t LocalDirectoryUpdater::fileWatchPeriod() const
{
return mDelayBetweenDirectoryUpdates ;
}

69
libretroshare/src/file_sharing/directory_updater.h Normal file
View file

@ -0,0 +1,69 @@
/*
* RetroShare C++ Directory parsing code.
*
* file_sharing/directory_updater.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
// This class crawls the given directry hierarchy and updates it. It does so by calling the
// shared file list source. This source may be of two types:
// - local: directories are crawled n disk and files are hashed / requested from a cache
// - remote: directories are requested remotely to a providing client
//
#include "file_sharing/hash_cache.h"
#include "file_sharing/directory_storage.h"
class LocalDirectoryUpdater: public HashStorageClient, public RsTickingThread
{
public:
LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ;
virtual ~LocalDirectoryUpdater() {}
void forceUpdate();
bool inDirectoryCheck() const ;
void setFileWatchPeriod(int seconds) ;
uint32_t fileWatchPeriod() const ;
void setEnabled(bool b) ;
bool isEnabled() const ;
protected:
virtual void data_tick() ;
virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size);
virtual bool hash_confirm(uint32_t client_param) ;
void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx);
void sweepSharedDirectories();
private:
HashStorage *mHashCache ;
LocalDirectoryStorage *mSharedDirectories ;
time_t mLastSweepTime;
time_t mLastTSUpdateTime;
uint32_t mDelayBetweenDirectoryUpdates;
bool mIsEnabled ;
};

44
libretroshare/src/file_sharing/file_sharing_defaults.h Normal file
View file

@ -0,0 +1,44 @@
/*
* RetroShare C++ File sharing default variables
*
* file_sharing/file_sharing_defaults.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes
static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 sec. Buy we only update for real if something has changed.
static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time
static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.
static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs.
static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this
static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission
static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days

232
libretroshare/src/file_sharing/filelist_io.cc Normal file
View file

@ -0,0 +1,232 @@
/*
* RetroShare File lists IO methods.
*
* file_sharing/filelist_io.h
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "retroshare/rsids.h"
#include "pqi/authssl.h"
#include "util/rsdir.h"
#include "serialiser/rsbaseserial.h"
#include "filelist_io.h"
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; }
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint64_t & val) { return setRawUInt64(buff,size,&offset,val) ; }
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; }
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const Sha1CheckSum & val) { return val.serialise(buff,size,offset) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint32_t & val) { return getRawUInt32(const_cast<uint8_t*>(buff),size,&offset,&val) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint64_t & val) { return getRawUInt64(const_cast<uint8_t*>(buff),size,&offset,&val) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,std::string & val) { return getRawString(const_cast<uint8_t*>(buff),size,&offset,val) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,Sha1CheckSum & val) { return val.deserialise(const_cast<uint8_t*>(buff),size,offset) ; }
template<> uint32_t FileListIO::serial_size(const uint32_t & ) { return 4 ; }
template<> uint32_t FileListIO::serial_size(const uint64_t & ) { return 8 ; }
template<> uint32_t FileListIO::serial_size(const std::string & val) { return getRawStringSize(val) ; }
template<> uint32_t FileListIO::serial_size(const Sha1CheckSum & ) { return Sha1CheckSum::serial_size(); }
bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size)
{
if(!checkSectionSize(buff,buff_size,offset,size))
return false;
if(!writeSectionHeader(buff,buff_size,offset,section_tag,size))
return false;
memcpy(&buff[offset],val,size) ;
offset += size ;
return true;
}
bool FileListIO::readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size)
{
if(!readSectionHeader(buff,buff_size,offset,check_section_tag,size))
return false;
val = (unsigned char *)rs_malloc(size) ;
if(!val)
return false;
memcpy(val,&buff[offset],size);
offset += size ;
return true ;
}
bool FileListIO::write125Size(unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t S)
{
if(S < 192)
{
if(offset+1 > data_size)
return false;
data[offset++] = (uint8_t)S ;
return true;
}
else if(S < 8384)
{
if(offset+2 > data_size)
return false;
data[offset+1] = (S - 192) & 0xff ;
data[offset ] = ((S - 192 - data[offset+1]) >> 8) + 192;
offset += 2 ;
return true;
}
else
{
if(offset+5 > data_size)
return false;
data[offset++] = 0xff ;
data[offset++] = (uint8_t)((S >> 24) & 255) ;
data[offset++] = (uint8_t)((S >> 16) & 255) ;
data[offset++] = (uint8_t)((S >> 8) & 255) ;
data[offset++] = (uint8_t)((S ) & 255) ;
return true ;
}
}
bool FileListIO::read125Size(const unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t& S)
{
if(offset + 1 >= data_size) return false;
uint8_t b1 = data[offset++] ;
if(b1 < 192)
{
S = b1;
return true ;
}
if(offset + 1 >= data_size) return false;
uint8_t b2 = data[offset++] ;
if(b1 < 224)
{
S = ((b1-192) << 8) + b2 + 192 ;
return true;
}
if(b1 != 0xff)
return false;
if(offset + 3 >= data_size) return false;
uint8_t b3 = data[offset++];
uint8_t b4 = data[offset++];
uint8_t b5 = data[offset++];
S = (b2 << 24) | (b3 << 16) | (b4 << 8) | b5 ;
return true;
}
bool FileListIO::saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size)
{
void *encryptedData = NULL ;
int encDataLen = 0 ;
if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,total_size, AuthSSL::getAuthSSL()->OwnId()))
{
std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl;
return false;
}
FILE *F = fopen( (fname+".tmp").c_str(),"wb" ) ;
if(!F)
{
std::cerr << "Cannot open encrypted file cache for writing: " << fname+".tmp" << std::endl;
free(encryptedData);
return false;
}
if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen)
{
std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl;
fclose(F) ;
free(encryptedData);
return false;
}
fclose(F) ;
RsDirUtil::renameFile(fname+".tmp",fname) ;
#ifdef FIM_DEBUG
std::cerr << "done." << std::endl ;
#endif
free(encryptedData);
return true;
}
bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size)
{
uint64_t file_size ;
if(!RsDirUtil::checkFile( fname,file_size,false ) )
{
std::cerr << "Encrypted hash cache file not present." << std::endl;
return false;
}
// read the binary stream into memory.
//
RsTemporaryMemory buffer(file_size) ;
if(buffer == NULL)
return false;
FILE *F = fopen( fname.c_str(),"rb") ;
if (!F)
{
std::cerr << "Cannot open file for reading encrypted file cache, filename " << fname << std::endl;
return false;
}
if(fread(buffer,1,file_size,F) != file_size)
{
std::cerr << "Cannot read from file " + fname << ": something's wrong." << std::endl;
fclose(F) ;
return false;
}
fclose(F) ;
// now decrypt
void *decrypted_data =NULL;
int decrypted_data_size =0;
if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size))
{
std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl;
return false;
}
data = (unsigned char*)decrypted_data ;
total_size = decrypted_data_size ;
return true;
}

145
libretroshare/src/file_sharing/filelist_io.h Normal file
View file

@ -0,0 +1,145 @@
/*
* RetroShare C++ File lists IO methods.
*
* file_sharing/filelist_io.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include "util/rsmemory.h"
// This file implements load/save of various fields used for file lists and directory content.
// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers.
static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ;
static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x01 ;
static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x10 ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x11 ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x12 ;
static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x13 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x20 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x21 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x22 ;
static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x30 ;
static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x31 ;
static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x32 ;
static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x40 ;
static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x41 ;
static const uint8_t FILE_LIST_IO_TAG_DIR_HASH = 0x50 ;
static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x51 ;
static const uint8_t FILE_LIST_IO_TAG_ROW = 0x60 ;
static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x61 ;
static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x62 ;
static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes)
class FileListIO
{
public:
template<typename T>
static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val)
{
uint32_t s = serial_size(val) ;
if(!checkSectionSize(buff,buff_size,offset,s))
return false;
if(!writeSectionHeader(buff,buff_size,offset,section_tag,s))
return false;
return serialise(buff,buff_size,offset,val) ;
}
template<typename T>
static bool readField(const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,T& val)
{
uint32_t section_size ;
if(!readSectionHeader(buff,buff_size,offset,check_section_tag,section_size))
return false;
return deserialise(buff,buff_size,offset,val);
}
static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ;
static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ;
template<class T> static bool serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const T& val) ;
template<class T> static bool deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,T& val) ;
template<class T> static uint32_t serial_size(const T& val) ;
static bool saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size);
static bool loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size);
private:
static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ;
static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ;
static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S)
{
if(offset + S + SECTION_HEADER_MAX_SIZE > buff_size)
{
buff = (unsigned char *)realloc(buff,offset + S + SECTION_HEADER_MAX_SIZE) ;
buff_size = offset + S + SECTION_HEADER_MAX_SIZE;
if(!buff)
return false ;
}
return true ;
}
static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,uint32_t S)
{
buff[offset++] = section_tag ;
if(!write125Size(buff,buff_size,offset,S)) return false ;
return true;
}
static bool readSectionHeader(const unsigned char *& buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,uint32_t& S)
{
if(offset + 1 > buff_size)
return false ;
uint8_t section_tag = buff[offset] ; // we do the offset++ after, only if the header can be read. Doing so, we can make multiple read attempts.
if(section_tag != check_section_tag)
return false;
offset++ ;
return read125Size(buff,buff_size,offset,S) ;
}
};

393
libretroshare/src/file_sharing/hash_cache.cc Normal file
View file

@ -0,0 +1,393 @@
/*
* RetroShare Hash cache
*
* file_sharing/hash_cache.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "util/rsdir.h"
#include "util/rsprint.h"
#include "rsserver/p3face.h"
#include "pqi/authssl.h"
#include "hash_cache.h"
#include "filelist_io.h"
#include "file_sharing_defaults.h"
//#define HASHSTORAGE_DEBUG 1
static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000;
static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000;
HashStorage::HashStorage(const std::string& save_file_name)
: mFilePath(save_file_name), mHashMtx("Hash Storage mutex")
{
mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME;
mRunning = false ;
mLastSaveTime = 0 ;
mTotalSizeToHash = 0;
mTotalFilesToHash = 0;
mMaxStorageDurationDays = DEFAULT_HASH_STORAGE_DURATION_DAYS ;
{
RS_STACK_MUTEX(mHashMtx) ;
locked_load() ;
}
}
static std::string friendlyUnit(uint64_t val)
{
const std::string units[5] = {"B","KB","MB","GB","TB"};
char buf[50] ;
double fact = 1.0 ;
for(unsigned int i=0; i<5; ++i)
if(double(val)/fact < 1024.0)
{
sprintf(buf,"%2.2f",double(val)/fact) ;
return std::string(buf) + " " + units[i];
}
else
fact *= 1024.0f ;
sprintf(buf,"%2.2f",double(val)/fact*1024.0f) ;
return std::string(buf) + " TB";
}
void HashStorage::data_tick()
{
FileHashJob job;
RsFileHash hash;
uint64_t size ;
{
bool empty ;
uint32_t st ;
{
RS_STACK_MUTEX(mHashMtx) ;
if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL))
{
locked_save();
mLastSaveTime = time(NULL) ;
mChanged = false ;
}
}
{
RS_STACK_MUTEX(mHashMtx) ;
empty = mFilesToHash.empty();
st = mInactivitySleepTime ;
}
// sleep off mutex!
if(empty)
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << "nothing to hash. Sleeping for " << st << " us" << std::endl;
#endif
usleep(st); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop.
if(st > MAX_INACTIVITY_SLEEP_TIME)
{
RS_STACK_MUTEX(mHashMtx) ;
mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME;
if(!mChanged) // otherwise it might prevent from saving the hash cache
{
std::cerr << "Stopping hashing thread." << std::endl;
shutdown();
mRunning = false ;
mTotalSizeToHash = 0;
mTotalFilesToHash = 0;
std::cerr << "done." << std::endl;
}
RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ;
}
else
{
RS_STACK_MUTEX(mHashMtx) ;
mInactivitySleepTime = 2*st ;
}
return ;
}
mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME;
{
RS_STACK_MUTEX(mHashMtx) ;
job = mFilesToHash.begin()->second ;
mFilesToHash.erase(mFilesToHash.begin()) ;
}
if(job.client->hash_confirm(job.client_param))
{
std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush();
std::string tmpout;
rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ;
RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ;
if(!RsDirUtil::getFileHash(job.full_path, hash,size, this))
std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl;
else
std::cerr << "done."<< std::endl;
// store the result
{
RS_STACK_MUTEX(mHashMtx) ;
HashStorageInfo& info(mFiles[job.full_path]);
info.filename = job.full_path ;
info.size = size ;
info.modf_stamp = job.ts ;
info.time_stamp = time(NULL);
info.hash = hash;
mChanged = true ;
++mHashCounter ;
mTotalHashedSize += size ;
}
}
}
// call the client
if(!hash.isNull())
job.client->hash_callback(job.client_param, job.full_path, hash, size);
}
bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param)
{
// check if the hash is up to date w.r.t. cache.
#ifdef HASHSTORAGE_DEBUG
std::cerr << "HASH Requested for file " << full_path << ": ";
#endif
RS_STACK_MUTEX(mHashMtx) ;
time_t now = time(NULL) ;
std::map<std::string,HashStorageInfo>::iterator it = mFiles.find(full_path) ;
// On windows we compare the time up to +/- 3600 seconds. This avoids re-hashing files in case of daylight saving change.
//
// See:
// https://support.microsoft.com/en-us/kb/190315
//
if(it != mFiles.end()
#ifdef WINDOWS_SYS
&& ( (uint64_t)mod_time == it->second.modf_stamp || (uint64_t)mod_time+3600 == it->second.modf_stamp ||(uint64_t)mod_time == it->second.modf_stamp+3600)
#else
&& (uint64_t)mod_time == it->second.modf_stamp
#endif
&& size == it->second.size)
{
it->second.time_stamp = now ;
#ifdef WINDOWS_SYS
if(it->second.time_stamp != (uint64_t)mod_time)
{
std::cerr << "(WW) detected a 1 hour shift in file modification time. This normally happens to many files at once, when daylight saving time shifts (file=\"" << full_path << "\")." << std::endl;
it->second.time_stamp = (uint64_t)mod_time;
}
#endif
known_hash = it->second.hash;
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Found in cache." << std::endl ;
#endif
return true ;
}
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Not in cache. Scheduling for re-hash." << std::endl ;
#endif
// we need to schedule a re-hashing
if(mFilesToHash.find(full_path) != mFilesToHash.end())
return false ;
FileHashJob job ;
job.client = c ;
job.size = size ;
job.client_param = client_param ;
job.full_path = full_path ;
job.ts = mod_time ;
mFilesToHash[full_path] = job;
mTotalSizeToHash += size ;
++mTotalFilesToHash;
if(!mRunning)
{
mRunning = true ;
std::cerr << "Starting hashing thread." << std::endl;
mHashCounter = 0;
mTotalHashedSize = 0;
start() ;
}
return false;
}
void HashStorage::clean()
{
RS_STACK_MUTEX(mHashMtx) ;
time_t now = time(NULL) ;
time_t duration = mMaxStorageDurationDays * 24 * 3600 ; // seconds
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Cleaning hash cache." << std::endl ;
#endif
for(std::map<std::string,HashStorageInfo>::iterator it(mFiles.begin());it!=mFiles.end();)
if(it->second.time_stamp + duration < (uint64_t)now)
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ;
#endif
std::map<std::string,HashStorageInfo>::iterator tmp(it) ;
++tmp ;
mFiles.erase(it) ;
it=tmp ;
mChanged = true ;
}
else
++it ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Done." << std::endl;
#endif
}
void HashStorage::locked_load()
{
unsigned char *data = NULL ;
uint32_t data_size=0;
if(!FileListIO::loadEncryptedDataFromFile(mFilePath,data,data_size))
{
std::cerr << "(EE) Cannot read hash cache." << std::endl;
return ;
}
uint32_t offset = 0 ;
HashStorageInfo info ;
uint32_t n=0;
while(offset < data_size)
if(readHashStorageInfo(data,data_size,offset,info))
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << info << std::endl;
++n ;
#endif
mFiles[info.filename] = info ;
}
free(data) ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << n << " entries loaded." << std::endl ;
#endif
}
void HashStorage::locked_save()
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ;
#endif
unsigned char *data = NULL ;
uint32_t offset = 0 ;
uint32_t total_size = 0;
for(std::map<std::string,HashStorageInfo>::const_iterator it(mFiles.begin());it!=mFiles.end();++it)
writeHashStorageInfo(data,total_size,offset,it->second) ;
if(!FileListIO::saveEncryptedDataToFile(mFilePath,data,offset))
{
std::cerr << "(EE) Cannot save hash cache data." << std::endl;
free(data) ;
return ;
}
std::cerr << mFiles.size() << " entries saved in hash cache." << std::endl;
free(data) ;
}
bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const
{
unsigned char *section_data = NULL ;
uint32_t section_size = 0;
uint32_t section_offset = 0;
// This way, the entire section is either read or skipped. That avoids the risk of being stuck somewhere in the middle
// of a section because of some unknown field, etc.
if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size))
return false;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ;
free(section_data);
return true;
}
bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const
{
unsigned char *section_data = NULL ;
uint32_t section_offset = 0 ;
uint32_t section_size = 0;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ;
// now write the whole string into a single section in the file
if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Writing hash storage section " << RsUtil::BinToHex(section_data,section_offset) << std::endl;
std::cerr << "Info.filename = " << info.filename << std::endl;
#endif
free(section_data) ;
return true;
}
std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& info)
{
return o << info.hash << " " << info.size << " " << info.filename ;
}

139
libretroshare/src/file_sharing/hash_cache.h Normal file
View file

@ -0,0 +1,139 @@
/*
* RetroShare C++ Hash cache.
*
* file_sharing/hash_cache.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <map>
#include "util/rsthreads.h"
#include "retroshare/rsfiles.h"
/*!
* \brief The HashStorageClient class
* Used by clients of the hash cache for receiving hash results when done. This is asynchrone of course since hashing
* might be quite costly.
*/
class HashStorageClient
{
public:
HashStorageClient() {}
virtual ~HashStorageClient() {}
// the result of the hashing info is sent to this method
virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)=0;
// this method is used to check that the client param is still valid just before hashing. This avoids hashing files
// that are still in queue while removed from shared lists.
virtual bool hash_confirm(uint32_t client_param)=0 ;
};
class HashStorage: public RsTickingThread
{
public:
HashStorage(const std::string& save_file_name) ;
/*!
* \brief requestHash Requests the hash for the given file, assuming size and mod_time are the same.
*
* \param full_path Full path to reach the file
* \param size Actual file size
* \param mod_time Actual file modification time
* \param known_hash Returned hash for the file.
* \param c Hash cache client to which the hash should be sent once calculated
* \param client_param Param to be passed to the client callback. Useful if the client needs a file ID.
*
* \return true if the supplied hash info is up to date.
*/
bool requestHash(const std::string& full_path, uint64_t size, time_t mod_time, RsFileHash& known_hash, HashStorageClient *c, uint32_t client_param) ;
struct HashStorageInfo
{
std::string filename ; // full path of the file
uint64_t size ;
uint32_t time_stamp ; // last time the hash was tested/requested
uint32_t modf_stamp ;
RsFileHash hash ;
} ;
// interaction with GUI, called from p3FileLists
void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } // duration for which the hash is kept even if the file is not shared anymore
uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; }
void clear() { mFiles.clear(); mChanged=true; } // drop all known hashes. Not something to do, except if you want to rehash the entire database
bool empty() const { return mFiles.empty() ; }
// Functions called by the thread
virtual void data_tick() ;
friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ;
private:
/*!
* \brief clean
* This function is responsible for removing old hashes, etc
*/
void clean() ;
// loading/saving the entire hash database to a file
void locked_save() ;
void locked_load() ;
bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const;
bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const;
// Local configuration and storage
uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries
std::map<std::string, HashStorageInfo> mFiles ; // stored as (full_path, hash_info)
std::string mFilePath ; // file where the hash database is stored
bool mChanged ;
struct FileHashJob
{
std::string full_path;
uint64_t size ;
HashStorageClient *client;
uint32_t client_param ;
time_t ts;
};
// current work
std::map<std::string,FileHashJob> mFilesToHash ;
// thread/mutex stuff
RsMutex mHashMtx ;
bool mRunning;
uint64_t mHashCounter;
uint32_t mInactivitySleepTime ;
uint64_t mTotalSizeToHash ;
uint64_t mTotalHashedSize ;
uint64_t mTotalFilesToHash ;
time_t mLastSaveTime ;
};

1445
libretroshare/src/file_sharing/p3filelists.cc Normal file
View file

File diff suppressed because it is too large Load diff

233
libretroshare/src/file_sharing/p3filelists.h Normal file
View file

@ -0,0 +1,233 @@
/*
* RetroShare C++ File lists service.
*
* file_sharing/p3filelists.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
//
// This class is responsible for
// - maintaining a list of shared file hierarchies for each known friends
// - talking to the GUI
// - providing handles for the directory tree listing GUI
// - providing search handles for FT
// - keeping these lists up to date
// - sending our own file list to friends depending on the defined access rights
// - serving file search requests from other services such as file transfer
//
// p3FileList does the following micro-tasks:
// - tick the watchers
// - get incoming info from the service layer, which can be:
// - directory content request => the directory content is shared to the friend
// - directory content => the directory watcher is notified
// - keep two queues of update requests:
// - fast queue that is handled in highest priority. This one is used for e.g. updating while browsing.
// - slow queue that is handled slowly. Used in background update of shared directories.
//
// The file lists are not directry updated. A FileListWatcher class is responsible for this
// in every case.
//
#pragma once
#include "ft/ftsearch.h"
#include "retroshare/rsfiles.h"
#include "services/p3service.h"
#include "file_sharing/hash_cache.h"
#include "file_sharing/directory_storage.h"
#include "pqi/p3cfgmgr.h"
#include "pqi/p3linkmgr.h"
class RemoteDirectoryUpdater ;
class LocalDirectoryUpdater ;
class RemoteDirectoryStorage ;
class LocalDirectoryStorage ;
class RsFileListsSyncRequestItem ;
class RsFileListsSyncResponseItem ;
class HashStorage ;
class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService
{
public:
typedef DirectoryStorage::EntryIndex EntryIndex; // this should probably be defined elsewhere
virtual RsServiceInfo getServiceInfo();
struct RsFileListSyncRequest
{
RsPeerId peerId ;
EntryIndex index ;
// [...] more to add here
};
p3FileDatabase(p3ServiceControl *mpeers) ;
~p3FileDatabase();
/*!
* \brief forceSyncWithPeers
*
* Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new group of friend is created, or when
* a friend was added/removed from a group.
*/
void forceSyncWithPeers() { NOT_IMPLEMENTED() ; }
// derived from p3Service
//
virtual int tick() ;
// ftSearch
virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const;
virtual int SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) ;
virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
// Interface for browsing dir hierarchy
//
void stopThreads() ;
void startThreads() ;
bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const;
// void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally.
void requestDirUpdate(void *ref) ; // triggers an update. Used when browsing.
int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ;
uint32_t getType(void *) const ;
// proxy method used by the web UI. Dont't delete!
int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const;
// set/update shared directories
void setSharedDirectories(const std::list<SharedDirInfo>& dirs);
void getSharedDirectories(std::list<SharedDirInfo>& dirs);
void updateShareFlags(const SharedDirInfo& info) ;
bool convertSharedFilePath(const std::string& path,std::string& fullpath);
// interface for hash caching
void setWatchPeriod(uint32_t seconds);
uint32_t watchPeriod() ;
void setWatchEnabled(bool b) ;
bool watchEnabled() ;
// interfact for directory parsing
void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed
bool inDirectoryCheck();
protected:
int filterResults(const std::list<EntryIndex>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const;
std::string makeRemoteFileName(const RsPeerId& pid) const;
// Derived from p3Config
//
virtual bool loadList(std::list<RsItem *>& items);
virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
virtual RsSerialiser *setupSerialiser() ;
void cleanup();
void tickRecv();
void tickSend();
private:
p3ServiceControl *mServCtrl ;
RsPeerId mOwnId ;
typedef uint64_t DirSyncRequestId ;
static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id, const RsFileHash &hash) ;
// utility functions to send items with some maximum size.
void splitAndSendItem(RsFileListsSyncResponseItem *ritem);
RsFileListsSyncResponseItem *recvAndRebuildItem(RsFileListsSyncResponseItem *ritem);
/*!
* \brief generateAndSendSyncRequest
* \param rds Remote directory storage for the request
* \param e Entry index to update
* \return true if the request is correctly sent.
*/
bool locked_generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e);
// File sync request queues. The fast one is used for online browsing when friends are connected.
// The slow one is used for background update of file lists.
//
std::map<DirSyncRequestId,RsFileListSyncRequest> mFastRequestQueue ;
std::map<DirSyncRequestId,RsFileListSyncRequest> mSlowRequestQueue ;
// Directory storage hierarchies
//
// The remote one is the reference for the PeerId index below:
// RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid)
std::vector<RemoteDirectoryStorage *> mRemoteDirectories ;
LocalDirectoryStorage *mLocalSharedDirs ;
LocalDirectoryUpdater *mLocalDirWatcher ;
// utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc
static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p);
static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ;
uint32_t locked_getFriendIndex(const RsPeerId& pid);
const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const;
void handleDirSyncRequest (RsFileListsSyncRequestItem *) ;
void handleDirSyncResponse (RsFileListsSyncResponseItem *) ;
std::map<RsPeerId,uint32_t> mFriendIndexMap ;
std::vector<RsPeerId> mFriendIndexTab;
// Directory synchronization
//
struct DirSyncRequestData
{
RsPeerId peer_id ;
time_t request_TS ;
uint32_t flags ;
};
time_t mLastRemoteDirSweepTS ; // TS for friend list update
std::map<DirSyncRequestId,DirSyncRequestData> mPendingSyncRequests ; // pending requests, waiting for an answer
std::map<DirSyncRequestId,RsFileListsSyncResponseItem *> mPartialResponseItems;
void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds, DirectoryStorage::EntryIndex e, int depth);
// We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name.
//
HashStorage *mHashCache ;
// Local flags and mutexes
mutable RsMutex mFLSMtx ;
uint32_t mUpdateFlags ;
std::string mFileSharingDir ;
time_t mLastCleanupTime;
};

363
libretroshare/src/file_sharing/rsfilelistitems.cc Normal file
View file

@ -0,0 +1,363 @@
/*
* RetroShare File lists service items
*
* file_sharing/rsfilelistsitems.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "serialiser/rsbaseserial.h"
#include "file_sharing/rsfilelistitems.h"
RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialise()" << std::endl;
#endif
/* get the type and size */
uint32_t rstype = getRsItemId(data);
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (getRsItemService(rstype) != RS_SERVICE_TYPE_FILE_DATABASE))
return NULL; /* wrong type */
switch(getRsItemSubType(rstype))
{
case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncRequestItem(data, size);
case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: return deserialFileListsSyncResponseItem(data, size);
// case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM: return deserialFileListsConfigItem (data, size);
default:
{
std::cerr << "(WW) RsFileListsSerialiser::deserialise() : unhandled item type " << getRsItemSubType(rstype) << std::endl;
return NULL;
}
}
}
uint32_t RsFileListsSerialiser::size(RsItem *item)
{
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
if(flst_item != NULL)
return flst_item->serial_size() ;
else
{
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
return 0;
}
}
bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size)
{
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
if(flst_item != NULL)
return flst_item->serialise(data,*size) ;
else
{
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
return 0;
}
}
bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const
{
tlvsize = serial_size() ;
offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
if(!setRsItemHeader(data, tlvsize, PacketId(), tlvsize))
{
std::cerr << "RsFileTransferItem::serialise_header(): ERROR. Not enough size!" << std::endl;
return false ;
}
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl;
#endif
offset += 8;
return true ;
}
bool RsFileListsSyncRequestItem::serialise(void *data, uint32_t& size) const
{
uint32_t tlvsize,offset=0;
bool ok = true;
if(!serialise_header(data,size,tlvsize,offset))
return false ;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl;
#endif
/* RsFileListsSyncMsgItem */
ok &= entry_hash.serialise(data, size, offset);
ok &= setRawUInt32(data, size, &offset, flags );
ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
ok &= setRawUInt64(data, size, &offset, request_id);
if(offset != tlvsize){
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl;
#endif
ok = false;
}
#ifdef RSSERIAL_DEBUG
if (!ok)
{
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl;
}
#endif
return ok;
}
bool RsFileListsSyncResponseItem::serialise(void *data, uint32_t& size) const
{
uint32_t tlvsize,offset=0;
bool ok = true;
if(!serialise_header(data,size,tlvsize,offset))
return false ;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl;
#endif
/* RsFileListsSyncMsgItem */
ok &= entry_hash.serialise(data, size, offset);
ok &= checksum.serialise(data, size, offset);
ok &= setRawUInt32(data, size, &offset, flags );
ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
ok &= setRawUInt64(data, size, &offset, request_id);
ok &= directory_content_data.SetTlv(data,size,&offset) ;
if(offset != tlvsize){
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl;
#endif
ok = false;
}
#ifdef RSSERIAL_DEBUG
if (!ok)
{
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl;
}
#endif
return ok;
}
//============================================================================================================================//
// Deserialisation //
//============================================================================================================================//
//RsFileListsConfigItem* RsFileListsSerialiser::deserialFileListsConfigItem(void *data, uint32_t *size)
//{
// NOT_IMPLEMENTED();
//
// return NULL ;
//}
RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestItem(void *data, uint32_t *size)
{
bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM);
uint32_t offset = 8;
RsFileListsSyncRequestItem* item = new RsFileListsSyncRequestItem();
ok &= item->entry_hash.deserialise(data, *size, offset);
ok &= getRawUInt32(data, *size, &offset, &item->flags);
ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
ok &= getRawUInt64(data, *size, &offset, &item->request_id);
if (offset != *size)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl;
#endif
/* error */
delete item;
return NULL;
}
if (!ok)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl;
#endif
delete item;
return NULL;
}
return item;
}
RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncResponseItem(void *data, uint32_t *size)
{
bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM);
uint32_t offset = 8;
RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem();
uint32_t entry_index ; // index of the directory to sync
uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
ok &= item->entry_hash.deserialise(data, *size, offset);
ok &= item->checksum.deserialise(data, *size, offset);
ok &= getRawUInt32(data, *size, &offset, &item->flags);
ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
ok &= getRawUInt64(data, *size, &offset, &item->request_id);
ok &= item->directory_content_data.GetTlv(data,*size,&offset) ;
if (offset != *size)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl;
#endif
/* error */
delete item;
return NULL;
}
if (!ok)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl;
#endif
delete item;
return NULL;
}
return item;
}
bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t subservice_type)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::checkItemHeader()" << std::endl;
#endif
/* get the type and size */
uint32_t rstype = getRsItemId(data);
uint32_t rssize = getRsItemSize(data);
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_FILE_DATABASE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype)))
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl;
#endif
return false; /* wrong type */
}
if (*size < rssize) /* check size */
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong size" << std::endl;
#endif
return false; /* not enough data */
}
/* set the packet length */
*size = rssize;
return true ;
}
//============================================================================================================================//
// Sizes //
//============================================================================================================================//
uint32_t RsFileListsSyncRequestItem::serial_size()const
{
uint32_t s = 8; //header size
s += RsFileHash::serial_size(); // entry hash
s += 4; // flags
s += 4; // last_known_recurs_modf_TS
s += 8; // request_id
return s;
}
uint32_t RsFileListsSyncResponseItem::serial_size()const
{
uint32_t s = 8; //header size
s += RsFileHash::serial_size(); // entry hash
s += RsFileHash::serial_size(); // checksum
s += 4; // flags
s += 4; // last_known_recurs_modf_TS
s += 8; // request_id
s += directory_content_data.TlvSize();
return s;
}
void RsFileListsSyncRequestItem::clear()
{
}
void RsFileListsSyncResponseItem::clear()
{
directory_content_data.TlvClear();
}
std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsFileListsSyncReqItem", indent);
uint16_t int_Indent = indent + 2;
printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl;
printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl;
printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl;
printRsItemEnd(out ,"RsFileListsSyncReqItem", indent);
return out;
}
std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsFileListsSyncDirItem", indent);
uint16_t int_Indent = indent + 2;
printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl;
printIndent(out , int_Indent); out << "Checksum : " << checksum << std::endl;
printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl;
printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl;
printIndent(out , int_Indent); out << "Data size: " << directory_content_data.bin_len << std::endl;
printRsItemEnd(out ,"RsFileListsSyncDirItem", indent);
return out;
}

135
libretroshare/src/file_sharing/rsfilelistitems.h Normal file
View file

@ -0,0 +1,135 @@
/*
* RetroShare File lists service items.
*
* file_sharing/rsfilelistitems.h
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <map>
#include <openssl/ssl.h>
#include "serialiser/rsserviceids.h"
#include "serialiser/rsserial.h"
#include "serialiser/rstlvbase.h"
#include "serialiser/rstlvitem.h"
#include "serialiser/rstlvkeys.h"
#include "gxs/rsgxsdata.h"
// These items have "flag type" numbers, but this is not used.
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01;
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM = 0x02;
const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03;
/*!
* Base class for filelist sync items
*/
class RsFileListsItem : public RsItem
{
public:
RsFileListsItem(uint8_t subtype)
: RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE, subtype)
{
setPriorityLevel(QOS_PRIORITY_RS_SLOW_SYNC_REQUEST); // this is the default. Someitems may be faster, on demand.
return;
}
virtual ~RsFileListsItem(){}
virtual bool serialise(void *data,uint32_t& size) const = 0 ;
virtual uint32_t serial_size() const = 0 ;
virtual void clear() = 0;
virtual std::ostream &print(std::ostream &out, uint16_t indent = 0) = 0;
bool serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const;
static const uint32_t FLAGS_SYNC_REQUEST = 0x0001 ;
static const uint32_t FLAGS_SYNC_RESPONSE = 0x0002 ;
static const uint32_t FLAGS_SYNC_DIR_CONTENT = 0x0004 ;
static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0008 ;
static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0010 ;
static const uint32_t FLAGS_SYNC_PARTIAL = 0x0020 ;
static const uint32_t FLAGS_SYNC_PARTIAL_END = 0x0040 ;
};
/*!
* Use to request synchronization on a specific directory. Also used to respond that the directory is up to date.
*/
class RsFileListsSyncRequestItem : public RsFileListsItem
{
public:
RsFileListsSyncRequestItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {}
virtual void clear();
virtual std::ostream &print(std::ostream &out, uint16_t indent);
virtual bool serialise(void *data,uint32_t& size) const;
virtual uint32_t serial_size() const ;
RsFileHash entry_hash ; // hash of the directory to sync
uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
};
class RsFileListsSyncResponseItem : public RsFileListsItem
{
public:
RsFileListsSyncResponseItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM) {}
virtual void clear();
virtual std::ostream &print(std::ostream &out, uint16_t indent);
virtual bool serialise(void *data,uint32_t& size) const;
virtual uint32_t serial_size() const ;
RsFileHash entry_hash ; // hash of the directory to sync
RsFileHash checksum ; // checksum of the bindary data, for checking
uint32_t flags; // is it a partial/final item (used for large items only)
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser.
};
class RsFileListsSerialiser : public RsSerialType
{
public:
RsFileListsSerialiser() : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {}
virtual ~RsFileListsSerialiser() {}
virtual uint32_t size(RsItem *item);
virtual bool serialise(RsItem *item, void *data, uint32_t *size);
virtual RsItem* deserialise(void *data, uint32_t *size);
private:
RsFileListsSyncRequestItem *deserialFileListsSyncRequestItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
RsFileListsSyncResponseItem *deserialFileListsSyncResponseItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
// RsFileListsSyncResponseItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type);
};

226
libretroshare/src/ft/ftcontroller.cc
View file

@ -73,10 +73,8 @@
static const int32_t SAVE_TRANSFERS_DELAY = 301 ; // save transfer progress every 301 seconds.
static const int32_t INACTIVE_CHUNKS_CHECK_DELAY = 240 ; // time after which an inactive chunk is released
static const int32_t MAX_TIME_INACTIVE_REQUEUED = 120 ; // time after which an inactive ftFileControl is bt-queued
static const int32_t TIMOUT_CACHE_FILE_TRANSFER = 800 ; // time after which cache transfer gets cancelled if inactive.
static const int32_t FT_FILECONTROL_QUEUE_ADD_END = 0 ;
static const int32_t FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE = 1 ;
const uint32_t FT_CNTRL_STANDARD_RATE = 10 * 1024 * 1024;
const uint32_t FT_CNTRL_SLOW_RATE = 100 * 1024;
@ -99,8 +97,8 @@ ftFileControl::ftFileControl(std::string fname,
return;
}
ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId)
:CacheTransfer(cs), p3Config(),
ftController::ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId)
: p3Config(),
last_save_time(0),
last_clean_time(0),
mDataplex(dm),
@ -231,7 +229,6 @@ void ftController::data_tick()
time_t now = time(NULL) ;
if(now > last_save_time + SAVE_TRANSFERS_DELAY)
{
cleanCacheDownloads() ;
searchForDirectSources() ;
IndicateConfigChanged() ;
@ -282,16 +279,15 @@ void ftController::searchForDirectSources()
for(std::map<RsFileHash,ftFileControl*>::iterator it(mDownloads.begin()); it != mDownloads.end(); ++it)
if(it->second->mState != ftFileControl::QUEUED && it->second->mState != ftFileControl::PAUSED)
if(! (it->second->mFlags & RS_FILE_REQ_CACHE))
{
FileInfo info ; // info needs to be re-allocated each time, to start with a clear list of peers (it's not cleared down there)
{
FileInfo info ; // info needs to be re-allocated each time, to start with a clear list of peers (it's not cleared down there)
if(mSearch->search(it->first, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info))
for(std::list<TransferInfo>::const_iterator pit = info.peers.begin(); pit != info.peers.end(); ++pit)
if(rsPeers->servicePermissionFlags(pit->peerId) & RS_NODE_PERM_DIRECT_DL)
if(it->second->mTransfer->addFileSource(pit->peerId)) /* if the sources don't exist already - add in */
setPeerState(it->second->mTransfer, pit->peerId, FT_CNTRL_STANDARD_RATE, mServiceCtrl->isPeerConnected(mFtServiceId, pit->peerId));
}
if(mSearch->search(it->first, RS_FILE_HINTS_REMOTE | RS_FILE_HINTS_SPEC_ONLY, info))
for(std::list<TransferInfo>::const_iterator pit = info.peers.begin(); pit != info.peers.end(); ++pit)
if(rsPeers->servicePermissionFlags(pit->peerId) & RS_NODE_PERM_DIRECT_DL)
if(it->second->mTransfer->addFileSource(pit->peerId)) /* if the sources don't exist already - add in */
setPeerState(it->second->mTransfer, pit->peerId, FT_CNTRL_STANDARD_RATE, mServiceCtrl->isPeerConnected(mFtServiceId, pit->peerId));
}
}
void ftController::tickTransfers()
@ -335,37 +331,6 @@ void ftController::setPriority(const RsFileHash& hash,DwlSpeed p)
it->second->mTransfer->setDownloadPriority(p) ;
}
void ftController::cleanCacheDownloads()
{
std::vector<RsFileHash> toCancel ;
time_t now = time(NULL) ;
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
for(std::map<RsFileHash,ftFileControl*>::iterator it(mDownloads.begin());it!=mDownloads.end();++it)
if (((it->second)->mFlags & RS_FILE_REQ_CACHE) && it->second->mState != ftFileControl::DOWNLOADING)
// check if a cache file is downloaded, if the case, timeout the transfer after TIMOUT_CACHE_FILE_TRANSFER
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::run() cache transfer found. age of this tranfer is :" << (int)(time(NULL) - (it->second)->mCreateTime);
std::cerr << std::endl;
#endif
if (now > (it->second)->mCreator->creationTimeStamp() + TIMOUT_CACHE_FILE_TRANSFER)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::run() cache transfer to old. Cancelling transfer. Hash :" << (it->second)->mHash << ", time=" << (it->second)->mCreateTime << ", now = " << time(NULL) ;
std::cerr << std::endl;
#endif
toCancel.push_back((it->second)->mHash);
}
}
}
for(uint32_t i=0;i<toCancel.size();++i)
FileCancel(toCancel[i]);
}
/* Called every 10 seconds or so */
void ftController::checkDownloadQueue()
{
@ -446,17 +411,13 @@ void ftController::checkDownloadQueue()
for(uint32_t p=0;p<_queue.size();++p)
{
if(p < _min_prioritized_transfers)
if(_queue[p]->mFlags & RS_FILE_REQ_CACHE) // cache file. add to potential move list
to_move_before.push_back(p) ;
else
++user_transfers ; // count one more user file in the prioritized range.
else
{
if(to_move_after.size() + user_transfers >= _min_prioritized_transfers) // we caught enough transfers to move back to the top of the queue.
break ;
if(!(_queue[p]->mFlags & RS_FILE_REQ_CACHE)) // non cache file. add to potential move list
to_move_after.push_back(p) ;
to_move_after.push_back(p) ;
}
}
uint32_t to_move = (uint32_t)std::max(0,(int)_min_prioritized_transfers - (int)user_transfers) ; // we move as many transfers as needed to get _min_prioritized_transfers user transfers.
@ -482,27 +443,6 @@ void ftController::locked_addToQueue(ftFileControl* ftfc,int add_strategy)
case FT_FILECONTROL_QUEUE_ADD_END: _queue.push_back(ftfc) ;
locked_checkQueueElement(_queue.size()-1) ;
break ;
case FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE:
{
// We add the transfer just before the first non cache transfer.
// This is costly, so only use this in case we really need it.
//
uint32_t pos =0;
while(pos < _queue.size() && (pos < _min_prioritized_transfers || (_queue[pos]->mFlags & RS_FILE_REQ_CACHE)>0) )
++pos ;
_queue.push_back(NULL) ;
for(int i=int(_queue.size())-1;i>(int)pos;--i)
{
_queue[i] = _queue[i-1] ;
locked_checkQueueElement(i) ;
}
_queue[pos] = ftfc ;
locked_checkQueueElement(pos) ;
}
break ;
}
}
@ -787,8 +727,8 @@ bool ftController::completeFile(const RsFileHash& hash)
std::string path;
std::string name;
uint64_t size = 0;
uint32_t state = 0;
uint32_t period = 0;
uint32_t state = 0;
uint32_t period = 0;
TransferRequestFlags flags ;
TransferRequestFlags extraflags ;
uint32_t completeCount = 0;
@ -889,12 +829,8 @@ bool ftController::completeFile(const RsFileHash& hash)
locked_queueRemove(it->second->mQueuePosition) ;
/* switch map */
if (!(fc->mFlags & RS_FILE_REQ_CACHE)) /* clean up completed cache files automatically */
{
mCompleted[fc->mHash] = fc;
completeCount = mCompleted.size();
} else
delete fc ;
mCompleted[fc->mHash] = fc;
completeCount = mCompleted.size();
mDownloads.erase(it);
@ -910,51 +846,14 @@ bool ftController::completeFile(const RsFileHash& hash)
/* If it has a callback - do it now */
if(flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA))// | RS_FILE_HINTS_MEDIA))
if(flags & RS_FILE_REQ_EXTRA)// | RS_FILE_HINTS_MEDIA))
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() doing Callback, callbackflags:" << (flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA ));//| RS_FILE_HINTS_MEDIA)) ;
std::cerr << std::endl;
#endif
if(flags & RS_FILE_REQ_CACHE)
{
/* callback */
if (state == ftFileControl::COMPLETED)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() doing Callback : Success";
std::cerr << std::endl;
#endif
CompletedCache(hash);
}
else
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() Cache Callback : Failed";
std::cerr << std::endl;
#endif
FailedCache(hash);
}
}
if(flags & RS_FILE_REQ_EXTRA)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() adding to ExtraList";
std::cerr << std::endl;
#endif
mExtraList->addExtraFile(path, hash, size, period, extraflags);
}
// if(flags & RS_FILE_HINTS_MEDIA)
// {
//#ifdef CONTROL_DEBUG
// std::cerr << "ftController::completeFile() NULL MEDIA callback";
// std::cerr << std::endl;
//#endif
// }
}
else
{
@ -965,14 +864,12 @@ bool ftController::completeFile(const RsFileHash& hash)
}
/* Notify GUI */
if ((flags & RS_FILE_REQ_CACHE) == 0) {
RsServer::notify()->AddPopupMessage(RS_POPUP_DOWNLOAD, hash.toStdString(), name, "");
RsServer::notify()->AddPopupMessage(RS_POPUP_DOWNLOAD, hash.toStdString(), name, "");
RsServer::notify()->notifyDownloadComplete(hash.toStdString());
RsServer::notify()->notifyDownloadCompleteCount(completeCount);
RsServer::notify()->notifyDownloadComplete(hash.toStdString());
RsServer::notify()->notifyDownloadCompleteCount(completeCount);
rsFiles->ForceDirectoryCheck() ;
}
rsFiles->ForceDirectoryCheck() ;
IndicateConfigChanged(); /* completed transfer -> save */
return true;
@ -1134,17 +1031,16 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash
// remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files.
//
if(!(flags & RS_FILE_REQ_CACHE))
for(std::list<RsPeerId>::iterator it = srcIds.begin(); it != srcIds.end(); )
if(!(rsPeers->servicePermissionFlags(*it) & RS_NODE_PERM_DIRECT_DL))
{
std::list<RsPeerId>::iterator tmp(it) ;
++tmp ;
srcIds.erase(it) ;
it = tmp ;
}
else
++it ;
for(std::list<RsPeerId>::iterator it = srcIds.begin(); it != srcIds.end(); )
if(!(rsPeers->servicePermissionFlags(*it) & RS_NODE_PERM_DIRECT_DL))
{
std::list<RsPeerId>::iterator tmp(it) ;
++tmp ;
srcIds.erase(it) ;
it = tmp ;
}
else
++it ;
std::list<RsPeerId>::const_iterator it;
std::list<TransferInfo>::const_iterator pit;
@ -1280,7 +1176,7 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash
if(flags & RS_FILE_REQ_ANONYMOUS_ROUTING)
mTurtle->monitorTunnels(hash,mFtServer,true) ;
bool assume_availability = flags & RS_FILE_REQ_CACHE ; // assume availability for cache files
bool assume_availability = false;
ftFileCreator *fc = new ftFileCreator(savepath, size, hash,assume_availability);
ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this);
@ -1911,58 +1807,6 @@ void ftController::statusChange(const std::list<pqiServicePeer> &plist)
}
}
/* Cache Interface */
bool ftController::RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::RequestCacheFile(" << id << ",";
std::cerr << path << "," << hash << "," << size << ")";
std::cerr << std::endl;
#endif
/* Request File */
std::list<RsPeerId> ids;
ids.push_back(id);
FileInfo info ;
if(mSearch->search(hash, RS_FILE_HINTS_CACHE, info))
{
#ifdef CONTROL_DEBUG
std::cerr << "I already have this file:" << std::endl ;
std::cerr << " path: " << info.path << std::endl ;
std::cerr << " fname: " << info.fname << std::endl ;
std::cerr << " hash: " << info.hash << std::endl ;
std::cerr << "Copying it !!" << std::endl ;
#endif
if(info.size > 0 && copyFile(info.path,path+"/"+hash.toStdString()))
{
CompletedCache(hash);
return true ;
}
else
return false ;
}
FileRequest(hash.toStdString(), hash, size, path, RS_FILE_REQ_CACHE | RS_FILE_REQ_NO_SEARCH, ids);
return true;
}
bool ftController::CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size)
{
std::cerr << "ftController::CancelCacheFile(" << id << ",";
std::cerr << path << "," << hash << "," << size << ")";
std::cerr << std::endl;
#ifdef CONTROL_DEBUG
#endif
return FileCancel(hash);
}
const std::string active_downloads_size_ss("MAX_ACTIVE_DOWNLOADS");
const std::string min_prioritized_downl_ss("MIN_PRORITIZED_DOWNLOADS");
const std::string download_dir_ss("DOWN_DIR");
@ -2056,14 +1900,6 @@ bool ftController::saveList(bool &cleanup, std::list<RsItem *>& saveData)
/* ignore cache files. As this is small files, better download them again from scratch at restart.*/
if (fit->second->mFlags & RS_FILE_REQ_CACHE)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftcontroller::saveList(): Not saving (callback) file entry " << fit->second->mName << ", " << fit->second->mHash << ", " << fit->second->mSize << std::endl ;
#endif
continue;
}
// Node: We still save finished transfers. This keeps transfers that are
// in checking mode. Finished or checked transfers will restart and
// immediately terminate/recheck at next startup.

9
libretroshare/src/ft/ftcontroller.h
View file

@ -48,7 +48,6 @@ class ftDataMultiplex;
class p3turtle ;
class p3ServiceControl;
#include "dbase/cachestrapper.h"
#include "util/rsthreads.h"
#include "pqi/pqiservicemonitor.h"
#include "pqi/p3cfgmgr.h"
@ -113,12 +112,12 @@ class ftPendingRequest
};
class ftController: public CacheTransfer, public RsTickingThread, public pqiServiceMonitor, public p3Config
class ftController: public RsTickingThread, public pqiServiceMonitor, public p3Config
{
public:
/* Setup */
ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId);
ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId);
void setFtSearchNExtra(ftSearch *, ftExtraList *);
void setTurtleRouter(p3turtle *) ;
@ -195,10 +194,6 @@ class ftController: public CacheTransfer, public RsTickingThread, public pqiServ
protected:
virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
void cleanCacheDownloads() ;
void searchForDirectSources() ;
void tickTransfers() ;

2
libretroshare/src/ft/ftdatamultiplex.cc
View file

@ -1104,8 +1104,6 @@ bool ftDataMultiplex::handleSearchRequest(const RsPeerId& peerId, const RsFileHa
if(rsTurtle->isTurtlePeer(peerId))
hintflags |= RS_FILE_HINTS_NETWORK_WIDE ;
else
hintflags |= RS_FILE_HINTS_CACHE ;
if(mSearch->search(hash, hintflags, info))
{

419
libretroshare/src/ft/ftserver.cc
View file

@ -31,6 +31,7 @@
#include "retroshare/rspeers.h"
const int ftserverzone = 29539;
#include "file_sharing/p3filelists.h"
#include "ft/ftturtlefiletransferitem.h"
#include "ft/ftserver.h"
#include "ft/ftextralist.h"
@ -43,11 +44,6 @@ const int ftserverzone = 29539;
#include "pqi/p3notify.h"
#include "rsserver/p3face.h"
// Includes CacheStrapper / FiMonitor / FiStore for us.
#include "ft/ftdbase.h"
#include "pqi/pqi.h"
#include "pqi/p3linkmgr.h"
@ -64,15 +60,11 @@ static const time_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priorit
/* Setup */
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc)
: p3Service(),
mPeerMgr(pm),
mServiceCtrl(sc),
mCacheStrapper(NULL),
mFiStore(NULL), mFiMon(NULL),
mPeerMgr(pm), mServiceCtrl(sc),
mFileDatabase(NULL),
mFtController(NULL), mFtExtra(NULL),
mFtDataplex(NULL), mFtSearch(NULL), srvMutex("ftServer")
{
mCacheStrapper = new ftCacheStrapper(sc, getServiceInfo().mServiceType);
addSerialType(new RsFileTransferSerialiser()) ;
}
@ -145,33 +137,25 @@ void ftServer::SetupFtServer()
mFtDataplex = new ftDataMultiplex(ownId, this, mFtSearch);
/* make Controller */
mFtController = new ftController(mCacheStrapper, mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType);
mFtController = new ftController(mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType);
mFtController -> setFtSearchNExtra(mFtSearch, mFtExtra);
std::string tmppath = ".";
mFtController->setPartialsDirectory(tmppath);
mFtController->setDownloadDirectory(tmppath);
/* Make Cache Source/Store */
mFiStore = new ftFiStore(mCacheStrapper, mFtController, mPeerMgr, ownId, remotecachedir);
mFiMon = new ftFiMonitor(mCacheStrapper,localcachedir, ownId,mConfigPath);
/* now add the set to the cachestrapper */
CachePair cp(mFiMon, mFiStore, CacheId(RS_SERVICE_TYPE_FILE_INDEX, 0));
mCacheStrapper -> addCachePair(cp);
/* complete search setup */
mFtSearch->addSearchMode(mCacheStrapper, RS_FILE_HINTS_CACHE);
mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA);
mFtSearch->addSearchMode(mFiMon, RS_FILE_HINTS_LOCAL);
mFtSearch->addSearchMode(mFiStore, RS_FILE_HINTS_REMOTE);
mServiceCtrl->registerServiceMonitor(mFtController, getServiceInfo().mServiceType);
mServiceCtrl->registerServiceMonitor(mCacheStrapper, getServiceInfo().mServiceType);
return;
}
void ftServer::connectToFileDatabase(p3FileDatabase *fdb)
{
mFileDatabase = fdb ;
mFtSearch->addSearchMode(fdb, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE);
}
void ftServer::connectToTurtleRouter(p3turtle *fts)
{
mTurtleRouter = fts ;
@ -193,7 +177,7 @@ void ftServer::StartupThreads()
/* startup Monitor Thread */
/* startup the FileMonitor (after cache load) */
/* start it up */
mFiMon->start("ft monitor");
mFileDatabase->startThreads();
/* Controller thread */
mFtController->start("ft ctrl");
@ -210,9 +194,6 @@ void ftServer::StopThreads()
/* stop Controller thread */
mFtController->join();
/* stop Monitor Thread */
mFiMon->join();
/* self contained threads */
/* stop ExtraList Thread */
mFtExtra->join();
@ -223,24 +204,15 @@ void ftServer::StopThreads()
delete (mFtController);
mFtController = NULL;
delete (mFiMon);
mFiMon = NULL;
delete (mFtExtra);
mFtExtra = NULL;
}
CacheStrapper *ftServer::getCacheStrapper()
{
return mCacheStrapper;
/* stop Monitor Thread */
mFileDatabase->stopThreads();
delete mFileDatabase;
mFileDatabase = NULL ;
}
CacheTransfer *ftServer::getCacheTransfer()
{
return mFtController;
}
/***************************************************************/
/********************** RsFiles Interface **********************/
/***************************************************************/
@ -256,11 +228,6 @@ bool ftServer::ResumeTransfers()
return true;
}
bool ftServer::checkHash(const RsFileHash& /*hash*/, std::string& /*error_string*/)
{
return true ;
}
bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data)
{
return mFtDataplex->getFileData(hash, offset, requested_size,data);
@ -268,20 +235,12 @@ bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& re
bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
{
return mFtController->alreadyHaveFile(hash, info);
return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info);
}
bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list<RsPeerId>& srcIds)
{
std::string error_string ;
if(!checkHash(hash,error_string))
{
RsServer::notify()->notifyErrorMsg(0,0,"Error handling hash \""+hash.toStdString()+"\". This hash appears to be invalid(Error string=\""+error_string+"\"). This is probably due an bad handling of strings.") ;
return false ;
}
std::cerr << "Requesting " << fname << std::endl ;
std::cerr << "Requesting " << fname << std::endl ;
if(!mFtController->FileRequest(fname, hash, size, dest, flags, srcIds))
return false ;
@ -335,14 +294,6 @@ bool ftServer::FileClearCompleted()
{
return mFtController->FileClearCompleted();
}
void ftServer::setMinPrioritizedTransfers(uint32_t s)
{
mFtController->setMinPrioritizedTransfers(s) ;
}
uint32_t ftServer::getMinPrioritizedTransfers()
{
return mFtController->getMinPrioritizedTransfers() ;
}
void ftServer::setQueueSize(uint32_t s)
{
mFtController->setQueueSize(s) ;
@ -381,7 +332,12 @@ bool ftServer::FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo&
return mFtController->getFileDownloadChunksDetails(hash,info);
}
/* Directory Handling */
void ftServer::requestDirUpdate(void *ref)
{
mFileDatabase->requestDirUpdate(ref) ;
}
/* Directory Handling */
void ftServer::setDownloadDirectory(std::string path)
{
mFtController->setDownloadDirectory(path);
@ -402,16 +358,18 @@ std::string ftServer::getPartialsDirectory()
return mFtController->getPartialsDirectory();
}
/***************************************************************/
/************************* Other Access ************************/
/***************************************************************/
/***************************************************************/
/************************* Other Access ************************/
/***************************************************************/
bool ftServer::copyFile(const std::string& source, const std::string& dest)
{
return mFtController->copyFile(source, dest);
}
void ftServer::FileDownloads(std::list<RsFileHash> &hashs)
{
mFtController->FileDownloads(hashs);
/* this only contains downloads.... not completed */
//return mFtDataplex->FileDownloads(hashs);
}
bool ftServer::FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& cmap)
@ -438,7 +396,7 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi
// file, we skip the call to fileDetails() for efficiency reasons.
//
FileInfo info2 ;
if( (!(info.transfer_info_flags & RS_FILE_REQ_CACHE)) && mFtController->FileDetails(hash, info2))
if(mFtController->FileDetails(hash, info2))
info.fname = info2.fname ;
return true ;
@ -539,12 +497,11 @@ bool ftServer::handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_i
return res ;
}
/***************************************************************/
/******************* ExtraFileList Access **********************/
/***************************************************************/
/***************************************************************/
/******************* ExtraFileList Access **********************/
/***************************************************************/
bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size,
uint32_t period, TransferRequestFlags flags)
bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags)
{
return mFtExtra->addExtraFile(fname, hash, size, period, flags);
}
@ -564,161 +521,87 @@ bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info)
return mFtExtra->hashExtraFileDone(localpath, info);
}
bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size,
std::string destpath)
bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath)
{
return mFtExtra->moveExtraFile(fname, hash, size, destpath);
}
/***************************************************************/
/******************** Directory Listing ************************/
/***************************************************************/
/***************************************************************/
/******************** Directory Listing ************************/
/***************************************************************/
int ftServer::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::RequestDirDetails(uid:" << uid;
std::cerr << ", path:" << path << ", ...) -> mFiStore";
std::cerr << std::endl;
if (!mFiStore)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
}
#endif
if(uid == mServiceCtrl->getOwnId())
return mFiMon->RequestDirDetails(path, details);
else
return mFiStore->RequestDirDetails(uid, path, details);
return mFileDatabase->RequestDirDetails(uid, path, details);
}
bool ftServer::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags)
{
return mFileDatabase->findChildPointer(ref,row,result,flags) ;
}
int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::RequestDirDetails(ref:" << ref;
std::cerr << ", flags:" << flags << ", ...) -> mFiStore";
std::cerr << std::endl;
if (!mFiStore)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
}
#endif
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->RequestDirDetails(ref, details, flags);
else
return mFiStore->RequestDirDetails(ref, details, flags);
return mFileDatabase->RequestDirDetails(ref,details,flags) ;
}
uint32_t ftServer::getType(void *ref, FileSearchFlags flags)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::RequestDirDetails(ref:" << ref;
std::cerr << ", flags:" << flags << ", ...) -> mFiStore";
std::cerr << std::endl;
if (!mFiStore)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
}
#endif
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->getType(ref);
else
return mFiStore->getType(ref);
return mFileDatabase->getType(ref) ;
}
/***************************************************************/
/******************** Search Interface *************************/
/***************************************************************/
/***************************************************************/
/******************** Search Interface *************************/
/***************************************************************/
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags)
{
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchKeywords(keywords, results,flags,RsPeerId());
else
return mFiStore->SearchKeywords(keywords, results,flags);
return 0 ;
return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId());
}
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::SearchKeywords()";
std::cerr << std::endl;
if (!mFiStore)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
}
#endif
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchKeywords(keywords, results,flags,peer_id);
else
return mFiStore->SearchKeywords(keywords, results,flags);
return mFileDatabase->SearchKeywords(keywords, results,flags,peer_id);
}
int ftServer::SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags)
int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags)
{
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchBoolExp(exp,results,flags,RsPeerId()) ;
else
return mFiStore->searchBoolExp(exp, results);
return 0 ;
return mFileDatabase->SearchBoolExp(exp, results,flags,RsPeerId());
}
int ftServer::SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
{
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchBoolExp(exp,results,flags,peer_id) ;
else
return mFiStore->searchBoolExp(exp, results);
return mFileDatabase->SearchBoolExp(exp,results,flags,peer_id) ;
}
/***************************************************************/
/*************** Local Shared Dir Interface ********************/
/***************************************************************/
bool ftServer::ConvertSharedFilePath(std::string path, std::string &fullpath)
bool ftServer::ConvertSharedFilePath(std::string path, std::string &fullpath)
{
return mFiMon->convertSharedFilePath(path, fullpath);
return mFileDatabase->convertSharedFilePath(path, fullpath);
}
void ftServer::updateSinceGroupPermissionsChanged()
{
mFiMon->forceDirListsRebuildAndSend();
mFileDatabase->forceSyncWithPeers();
}
void ftServer::ForceDirectoryCheck()
{
mFiMon->forceDirectoryCheck();
mFileDatabase->forceDirectoryCheck();
return;
}
bool ftServer::InDirectoryCheck()
{
return mFiMon->inDirectoryCheck();
}
bool ftServer::copyFile(const std::string& source, const std::string& dest)
{
return mFtController->copyFile(source, dest);
return mFileDatabase->inDirectoryCheck();
}
bool ftServer::getSharedDirectories(std::list<SharedDirInfo> &dirs)
{
mFiMon->getSharedDirectories(dirs);
mFileDatabase->getSharedDirectories(dirs);
return true;
}
bool ftServer::setSharedDirectories(std::list<SharedDirInfo> &dirs)
{
mFiMon->setSharedDirectories(dirs);
mFileDatabase->setSharedDirectories(dirs);
return true;
}
@ -728,7 +611,7 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir)
_dir.filename = RsDirUtil::convertPathToUnix(_dir.filename);
std::list<SharedDirInfo> dirList;
mFiMon->getSharedDirectories(dirList);
mFileDatabase->getSharedDirectories(dirList);
// check that the directory is not already in the list.
for(std::list<SharedDirInfo>::const_iterator it(dirList.begin());it!=dirList.end();++it)
@ -738,13 +621,13 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir)
// ok then, add the shared directory.
dirList.push_back(_dir);
mFiMon->setSharedDirectories(dirList);
mFileDatabase->setSharedDirectories(dirList);
return true;
}
bool ftServer::updateShareFlags(const SharedDirInfo& info)
{
mFiMon->updateShareFlags(info);
mFileDatabase->updateShareFlags(info);
return true ;
}
@ -761,7 +644,7 @@ bool ftServer::removeSharedDirectory(std::string dir)
std::cerr << std::endl;
#endif
mFiMon->getSharedDirectories(dirList);
mFileDatabase->getSharedDirectories(dirList);
#ifdef SERVER_DEBUG
for(it = dirList.begin(); it != dirList.end(); ++it)
@ -776,12 +659,7 @@ bool ftServer::removeSharedDirectory(std::string dir)
if(it == dirList.end())
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::removeSharedDirectory()";
std::cerr << " Cannot Find Directory... Fail";
std::cerr << std::endl;
#endif
std::cerr << "(EE) ftServer::removeSharedDirectory(): Cannot Find Directory... Fail" << std::endl;
return false;
}
@ -793,44 +671,20 @@ bool ftServer::removeSharedDirectory(std::string dir)
#endif
dirList.erase(it);
mFiMon->setSharedDirectories(dirList);
mFileDatabase->setSharedDirectories(dirList);
return true;
}
void ftServer::setWatchPeriod(int minutes)
{
mFiMon->setWatchPeriod(minutes*60) ;
}
int ftServer::watchPeriod() const
{
return mFiMon->watchPeriod()/60 ;
}
bool ftServer::watchEnabled() { return mFileDatabase->watchEnabled() ; }
int ftServer::watchPeriod() const { return mFileDatabase->watchPeriod()/60 ; }
void ftServer::setRememberHashFiles(bool b)
{
mFiMon->setRememberHashCache(b) ;
}
bool ftServer::rememberHashFiles() const
{
return mFiMon->rememberHashCache() ;
}
void ftServer::setRememberHashFilesDuration(uint32_t days)
{
mFiMon->setRememberHashCacheDuration(days) ;
}
uint32_t ftServer::rememberHashFilesDuration() const
{
return mFiMon->rememberHashCacheDuration() ;
}
void ftServer::clearHashCache()
{
mFiMon->clearHashCache() ;
}
void ftServer::setWatchEnabled(bool b) { mFileDatabase->setWatchEnabled(b) ; }
void ftServer::setWatchPeriod(int minutes) { mFileDatabase->setWatchPeriod(minutes*60) ; }
bool ftServer::getShareDownloadDirectory()
{
std::list<SharedDirInfo> dirList;
mFiMon->getSharedDirectories(dirList);
mFileDatabase->getSharedDirectories(dirList);
std::string dir = mFtController->getDownloadDirectory();
@ -844,7 +698,8 @@ bool ftServer::getShareDownloadDirectory()
bool ftServer::shareDownloadDirectory(bool share)
{
if (share) {
if (share)
{
/* Share */
SharedDirInfo inf ;
inf.filename = mFtController->getDownloadDirectory();
@ -852,48 +707,22 @@ bool ftServer::shareDownloadDirectory(bool share)
return addSharedDirectory(inf);
}
/* Unshare */
std::string dir = mFtController->getDownloadDirectory();
return removeSharedDirectory(dir);
else
{
/* Unshare */
std::string dir = mFtController->getDownloadDirectory();
return removeSharedDirectory(dir);
}
}
/***************************************************************/
/****************** End of RsFiles Interface *******************/
/***************************************************************/
/***************************************************************/
/**************** Config Interface *****************************/
/***************************************************************/
/* Key Functions to be overloaded for Full Configuration */
RsSerialiser *ftServer::setupSerialiser()
{
RsSerialiser *rss = new RsSerialiser ;
rss->addSerialType(new RsFileTransferSerialiser) ;
//rss->addSerialType(new RsGeneralConfigSerialiser());
return rss ;
}
bool ftServer::saveList(bool &/*cleanup*/, std::list<RsItem *>& /*list*/)
{
return true;
}
bool ftServer::loadList(std::list<RsItem *>& /*load*/)
{
return true;
}
bool ftServer::loadConfigMap(std::map<std::string, std::string> &/*configMap*/)
{
return true;
}
//bool ftServer::loadConfigMap(std::map<std::string, std::string> &/*configMap*/)
//{
// return true;
//}
/***************************************************************/
/********************** Data Flow **********************/
@ -1257,9 +1086,6 @@ int ftServer::tick()
if(handleIncoming())
moreToTick = true;
if(handleCacheData())
moreToTick = true;
static time_t last_law_priority_tasks_handling_time = 0 ;
time_t now = time(NULL) ;
@ -1275,45 +1101,6 @@ int ftServer::tick()
return moreToTick;
}
bool ftServer::handleCacheData()
{
std::list<std::pair<RsPeerId, RsCacheData> > cacheUpdates;
std::list<std::pair<RsPeerId, RsCacheData> >::iterator it;
int i=0 ;
#ifdef SERVER_DEBUG_CACHE
std::cerr << "handleCacheData() " << std::endl;
#endif
mCacheStrapper->getCacheUpdates(cacheUpdates);
for(it = cacheUpdates.begin(); it != cacheUpdates.end(); ++it)
{
/* construct reply */
RsFileTransferCacheItem *ci = new RsFileTransferCacheItem();
/* id from incoming */
ci -> PeerId(it->first);
ci -> file.hash = (it->second).hash;
ci -> file.name = (it->second).name;
ci -> file.path = ""; // (it->second).path;
ci -> file.filesize = (it->second).size;
ci -> cacheType = (it->second).cid.type;
ci -> cacheSubId = (it->second).cid.subid;
#ifdef SERVER_DEBUG_CACHE
std::string out2 = "Outgoing CacheStrapper Update -> RsCacheItem:\n";
ci -> print_string(out2);
std::cerr << out2 << std::endl;
#endif
sendItem(ci);
i++;
}
return i>0 ;
}
int ftServer::handleIncoming()
{
// now File Input.
@ -1411,42 +1198,6 @@ int ftServer::handleIncoming()
}
}
break ;
case RS_PKT_SUBTYPE_FT_CACHE_ITEM:
{
RsFileTransferCacheItem *ci = dynamic_cast<RsFileTransferCacheItem*>(item) ;
if (ci)
{
#ifdef SERVER_DEBUG_CACHE
std::cerr << "ftServer::handleIncoming: received cache item hash=" << ci->file.hash << ". from peer " << ci->PeerId() << std::endl;
#endif
/* these go to the CacheStrapper! */
RsCacheData data;
data.pid = ci->PeerId();
data.cid = CacheId(ci->cacheType, ci->cacheSubId);
data.path = ci->file.path;
data.name = ci->file.name;
data.hash = ci->file.hash;
data.size = ci->file.filesize;
data.recvd = time(NULL) ;
mCacheStrapper->recvCacheResponse(data, time(NULL));
}
}
break ;
// case RS_PKT_SUBTYPE_FT_CACHE_REQUEST:
// {
// // do nothing
// RsFileTransferCacheRequestItem *cr = dynamic_cast<RsFileTransferCacheRequestItem*>(item) ;
// if (cr)
// {
//#ifdef SERVER_DEBUG_CACHE
// std::cerr << "ftServer::handleIncoming: received cache request from peer " << cr->PeerId() << std::endl;
//#endif
// }
// }
// break ;
}
delete item ;
@ -1465,7 +1216,7 @@ int ftServer::handleIncoming()
bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
{
/* add all the subbits to config mgr */
cfgmgr->addConfiguration("ft_shared.cfg", mFiMon);
cfgmgr->addConfiguration("ft_database.cfg", mFileDatabase);
cfgmgr->addConfiguration("ft_extra.cfg", mFtExtra);
cfgmgr->addConfiguration("ft_transfers.cfg", mFtController);

339
libretroshare/src/ft/ftserver.h
View file

@ -47,17 +47,13 @@
#include "turtle/turtleclientservice.h"
#include "services/p3service.h"
#include "retroshare/rsfiles.h"
//#include "dbase/cachestrapper.h"
#include "pqi/pqi.h"
#include "pqi/p3cfgmgr.h"
class p3ConnectMgr;
class p3FileDatabase;
class CacheStrapper;
class CacheTransfer;
class ftCacheStrapper;
class ftFiStore;
class ftFiMonitor;
@ -70,230 +66,201 @@ class p3turtle;
class p3PeerMgr;
class p3ServiceControl;
class p3FileDatabase;
class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService
{
public:
public:
/***************************************************************/
/******************** Setup ************************************/
/***************************************************************/
/***************************************************************/
/******************** Setup ************************************/
/***************************************************************/
ftServer(p3PeerMgr *peerMgr, p3ServiceControl *serviceCtrl);
virtual RsServiceInfo getServiceInfo();
ftServer(p3PeerMgr *peerMgr, p3ServiceControl *serviceCtrl);
virtual RsServiceInfo getServiceInfo();
/* Assign important variables */
void setConfigDirectory(std::string path);
/* Assign important variables */
void setConfigDirectory(std::string path);
/* add Config Items (Extra, Controller) */
void addConfigComponents(p3ConfigMgr *mgr);
/* add Config Items (Extra, Controller) */
void addConfigComponents(p3ConfigMgr *mgr);
virtual CacheStrapper *getCacheStrapper();
virtual CacheTransfer *getCacheTransfer();
const RsPeerId& OwnId();
const RsPeerId& OwnId();
/* Final Setup (once everything is assigned) */
void SetupFtServer() ;
virtual void connectToTurtleRouter(p3turtle *p) ;
virtual void connectToFileDatabase(p3FileDatabase *b);
/* Final Setup (once everything is assigned) */
void SetupFtServer() ;
virtual void connectToTurtleRouter(p3turtle *p) ;
// Implements RsTurtleClientService
//
virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ;
virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
virtual RsTurtleGenericTunnelItem *deserialiseItem(void *data,uint32_t size) const ;
// Checks that the given hash is well formed. Used to chase
// string bugs.
static bool checkHash(const RsFileHash& hash,std::string& error_string) ;
void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;
// Implements RsTurtleClientService
//
virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ;
virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
virtual RsTurtleGenericTunnelItem *deserialiseItem(void *data,uint32_t size) const ;
/***************************************************************/
/*************** Control Interface *****************************/
/************** (Implements RsFiles) ***************************/
/***************************************************************/
void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;
void StartupThreads();
void StopThreads();
/***************************************************************/
/*************** Control Interface *****************************/
/************** (Implements RsFiles) ***************************/
/***************************************************************/
// member access
void StartupThreads();
void StopThreads();
ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; }
ftController *getController() const { return mFtController ; }
// member access
ftDataMultiplex *getMultiplexer() const { return mFtDataplex ; }
ftController *getController() const { return mFtController ; }
/**
/**
* @see RsFiles::getFileData
*/
bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data);
bool getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data);
/***
* Control of Downloads
***/
virtual bool alreadyHaveFile(const RsFileHash& hash, FileInfo &info);
virtual bool FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list<RsPeerId>& srcIds);
virtual bool FileCancel(const RsFileHash& hash);
virtual bool FileControl(const RsFileHash& hash, uint32_t flags);
virtual bool FileClearCompleted();
virtual bool setDestinationDirectory(const RsFileHash& hash,const std::string& new_path) ;
virtual bool setDestinationName(const RsFileHash& hash,const std::string& new_name) ;
virtual bool setChunkStrategy(const RsFileHash& hash,FileChunksInfo::ChunkStrategy s) ;
virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ;
virtual FileChunksInfo::ChunkStrategy defaultChunkStrategy() ;
virtual uint32_t freeDiskSpaceLimit() const ;
virtual void setFreeDiskSpaceLimit(uint32_t size_in_mb) ;
/***
* Control of Downloads
***/
virtual bool alreadyHaveFile(const RsFileHash& hash, FileInfo &info);
virtual bool FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list<RsPeerId>& srcIds);
virtual bool FileCancel(const RsFileHash& hash);
virtual bool FileControl(const RsFileHash& hash, uint32_t flags);
virtual bool FileClearCompleted();
virtual bool setDestinationDirectory(const RsFileHash& hash,const std::string& new_path) ;
virtual bool setDestinationName(const RsFileHash& hash,const std::string& new_name) ;
virtual bool setChunkStrategy(const RsFileHash& hash,FileChunksInfo::ChunkStrategy s) ;
virtual void setDefaultChunkStrategy(FileChunksInfo::ChunkStrategy) ;
virtual FileChunksInfo::ChunkStrategy defaultChunkStrategy() ;
virtual uint32_t freeDiskSpaceLimit() const ;
virtual void setFreeDiskSpaceLimit(uint32_t size_in_mb) ;
/***
* Control of Downloads Priority.
***/
virtual uint32_t getMinPrioritizedTransfers() ;
virtual void setMinPrioritizedTransfers(uint32_t s) ;
virtual uint32_t getQueueSize() ;
virtual void setQueueSize(uint32_t s) ;
virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv);
virtual bool changeDownloadSpeed(const RsFileHash& hash, int speed);
virtual bool getDownloadSpeed(const RsFileHash& hash, int & speed);
virtual bool clearDownload(const RsFileHash& hash);
//virtual void getDwlDetails(std::list<DwlDetails> & details);
/***
* Control of Downloads Priority.
***/
virtual uint32_t getQueueSize() ;
virtual void setQueueSize(uint32_t s) ;
virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv);
virtual bool changeDownloadSpeed(const RsFileHash& hash, int speed);
virtual bool getDownloadSpeed(const RsFileHash& hash, int & speed);
virtual bool clearDownload(const RsFileHash& hash);
//virtual void getDwlDetails(std::list<DwlDetails> & details);
/***
* Download/Upload Details
***/
virtual void FileDownloads(std::list<RsFileHash> &hashs);
virtual bool FileUploads(std::list<RsFileHash> &hashs);
virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info);
virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ;
virtual bool FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& map) ;
/***
* Download/Upload Details
***/
virtual void FileDownloads(std::list<RsFileHash> &hashs);
virtual bool FileUploads(std::list<RsFileHash> &hashs);
virtual bool FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info);
virtual bool FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo& info) ;
virtual bool FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& map) ;
/***
* Extra List Access
***/
virtual bool ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags);
virtual bool ExtraFileRemove(const RsFileHash& hash, TransferRequestFlags flags);
virtual bool ExtraFileHash(std::string localpath, uint32_t period, TransferRequestFlags flags);
virtual bool ExtraFileStatus(std::string localpath, FileInfo &info);
virtual bool ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath);
/***
* Extra List Access
***/
virtual bool ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags);
virtual bool ExtraFileRemove(const RsFileHash& hash, TransferRequestFlags flags);
virtual bool ExtraFileHash(std::string localpath, uint32_t period, TransferRequestFlags flags);
virtual bool ExtraFileStatus(std::string localpath, FileInfo &info);
virtual bool ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath);
/***
* Directory Listing / Search Interface
***/
virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details);
virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags);
virtual uint32_t getType(void *ref,FileSearchFlags flags) ;
/***
* Directory Listing / Search Interface
***/
virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details);
virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags);
virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) ;
virtual uint32_t getType(void *ref,FileSearchFlags flags) ;
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags);
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags);
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags);
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags);
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
/***
* Utility Functions
***/
virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath);
virtual void ForceDirectoryCheck();
virtual void updateSinceGroupPermissionsChanged() ;
virtual bool InDirectoryCheck();
virtual bool copyFile(const std::string& source, const std::string& dest);
/***
* Utility Functions
***/
virtual bool ConvertSharedFilePath(std::string path, std::string &fullpath);
virtual void ForceDirectoryCheck();
virtual void updateSinceGroupPermissionsChanged() ;
virtual bool InDirectoryCheck();
virtual bool copyFile(const std::string& source, const std::string& dest);
/***
* Directory Handling
***/
virtual void setDownloadDirectory(std::string path);
virtual void setPartialsDirectory(std::string path);
virtual std::string getDownloadDirectory();
virtual std::string getPartialsDirectory();
/***
* Directory Handling
***/
virtual void requestDirUpdate(void *ref) ; // triggers the update of the given reference. Used when browsing.
virtual void setDownloadDirectory(std::string path);
virtual void setPartialsDirectory(std::string path);
virtual std::string getDownloadDirectory();
virtual std::string getPartialsDirectory();
virtual bool getSharedDirectories(std::list<SharedDirInfo> &dirs);
virtual bool setSharedDirectories(std::list<SharedDirInfo> &dirs);
virtual bool addSharedDirectory(const SharedDirInfo& dir);
virtual bool updateShareFlags(const SharedDirInfo& dir); // updates the flags. The directory should already exist !
virtual bool removeSharedDirectory(std::string dir);
virtual bool getSharedDirectories(std::list<SharedDirInfo> &dirs);
virtual bool setSharedDirectories(std::list<SharedDirInfo> &dirs);
virtual bool addSharedDirectory(const SharedDirInfo& dir);
virtual bool updateShareFlags(const SharedDirInfo& dir); // updates the flags. The directory should already exist !
virtual bool removeSharedDirectory(std::string dir);
virtual bool getShareDownloadDirectory();
virtual bool shareDownloadDirectory(bool share);
virtual bool getShareDownloadDirectory();
virtual bool shareDownloadDirectory(bool share);
virtual void setRememberHashFilesDuration(uint32_t days) ;
virtual uint32_t rememberHashFilesDuration() const ;
virtual bool rememberHashFiles() const ;
virtual void setRememberHashFiles(bool) ;
virtual void clearHashCache() ;
virtual void setWatchPeriod(int minutes) ;
virtual int watchPeriod() const ;
virtual void setWatchPeriod(int minutes) ;
virtual int watchPeriod() const ;
virtual void setWatchEnabled(bool b) ;
virtual bool watchEnabled() ;
/***************************************************************/
/*************** Control Interface *****************************/
/***************************************************************/
/***************************************************************/
/*************** Data Transfer Interface ***********************/
/***************************************************************/
public:
virtual bool sendData(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
virtual bool sendDataRequest(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
virtual bool sendChunkMapRequest(const RsPeerId& peer_id,const RsFileHash& hash,bool is_client) ;
virtual bool sendChunkMap(const RsPeerId& peer_id,const RsFileHash& hash,const CompressedChunkMap& cmap,bool is_client) ;
virtual bool sendSingleChunkCRCRequest(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number) ;
virtual bool sendSingleChunkCRC(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ;
/***************************************************************/
/*************** Data Transfer Interface ***********************/
/***************************************************************/
public:
virtual bool sendData(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize, void *data);
virtual bool sendDataRequest(const RsPeerId& peerId, const RsFileHash& hash, uint64_t size, uint64_t offset, uint32_t chunksize);
virtual bool sendChunkMapRequest(const RsPeerId& peer_id,const RsFileHash& hash,bool is_client) ;
virtual bool sendChunkMap(const RsPeerId& peer_id,const RsFileHash& hash,const CompressedChunkMap& cmap,bool is_client) ;
virtual bool sendSingleChunkCRCRequest(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number) ;
virtual bool sendSingleChunkCRC(const RsPeerId& peer_id,const RsFileHash& hash,uint32_t chunk_number,const Sha1CheckSum& crc) ;
/*************** Internal Transfer Fns *************************/
virtual int tick();
/*************** Internal Transfer Fns *************************/
virtual int tick();
/* Configuration */
bool addConfiguration(p3ConfigMgr *cfgmgr);
bool ResumeTransfers();
/* Configuration */
bool addConfiguration(p3ConfigMgr *cfgmgr);
bool ResumeTransfers();
/*************************** p3 Config Overload ********************/
/******************* p3 Config Overload ************************/
protected:
/* Key Functions to be overloaded for Full Configuration */
virtual RsSerialiser *setupSerialiser();
virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
virtual bool loadList(std::list<RsItem *>& load);
protected:
int handleIncoming() ;
bool handleCacheData() ;
private:
bool loadConfigMap(std::map<std::string, std::string> &configMap);
/******************* p3 Config Overload ************************/
private:
/*************************** p3 Config Overload ********************/
/**** INTERNAL FUNCTIONS ***/
//virtual int reScanDirs();
//virtual int check_dBUpdate();
protected:
int handleIncoming() ;
bool handleCacheData() ;
private:
private:
/* no need for Mutex protection -
* as each component is protected independently.
*/
p3PeerMgr *mPeerMgr;
p3ServiceControl *mServiceCtrl;
p3FileDatabase *mFileDatabase ;
ftController *mFtController;
ftExtraList *mFtExtra;
ftDataMultiplex *mFtDataplex;
p3turtle *mTurtleRouter ;
ftFileSearch *mFtSearch;
/**** INTERNAL FUNCTIONS ***/
//virtual int reScanDirs();
//virtual int check_dBUpdate();
private:
/* no need for Mutex protection -
* as each component is protected independently.
*/
p3PeerMgr *mPeerMgr;
p3ServiceControl *mServiceCtrl;
ftCacheStrapper *mCacheStrapper;
ftFiStore *mFiStore;
ftFiMonitor *mFiMon;
ftController *mFtController;
ftExtraList *mFtExtra;
ftDataMultiplex *mFtDataplex;
p3turtle *mTurtleRouter ;
ftFileSearch *mFtSearch;
RsMutex srvMutex;
std::string mConfigPath;
std::string mDownloadPath;
std::string mPartialsPath;
RsMutex srvMutex;
std::string mConfigPath;
std::string mDownloadPath;
std::string mPartialsPath;
};

41
libretroshare/src/libretroshare.pro
View file

@ -37,6 +37,28 @@ debug {
QMAKE_CXXFLAGS *= -g -fno-omit-frame-pointer
}
CONFIG += file_lists
file_lists {
HEADERS *= file_sharing/p3filelists.h \
file_sharing/hash_cache.h \
file_sharing/filelist_io.h \
file_sharing/directory_storage.h \
file_sharing/directory_updater.h \
file_sharing/rsfilelistitems.h \
file_sharing/dir_hierarchy.h \
file_sharing/file_sharing_defaults.h
SOURCES *= file_sharing/p3filelists.cc \
file_sharing/hash_cache.cc \
file_sharing/filelist_io.cc \
file_sharing/directory_storage.cc \
file_sharing/directory_updater.cc \
file_sharing/dir_hierarchy.cc \
file_sharing/rsfilelistitems.cc
}
dsdv {
DEFINES *= SERVICES_DSDV
HEADERS += services/p3dsdv.h \
@ -345,16 +367,10 @@ INCLUDEPATH *= $${OPENPGPSDK_DIR}
PRE_TARGETDEPS *= $${OPENPGPSDK_DIR}/lib/libops.a
LIBS *= $${OPENPGPSDK_DIR}/lib/libops.a -lbz2
HEADERS += dbase/cachestrapper.h \
dbase/fimonitor.h \
dbase/findex.h \
dbase/fistore.h
HEADERS += ft/ftchunkmap.h \
ft/ftcontroller.h \
ft/ftdata.h \
ft/ftdatamultiplex.h \
ft/ftdbase.h \
ft/ftextralist.h \
ft/ftfilecreator.h \
ft/ftfileprovider.h \
@ -364,6 +380,10 @@ HEADERS += ft/ftchunkmap.h \
ft/fttransfermodule.h \
ft/ftturtlefiletransferitem.h
HEADERS += directory_updater.h \
directory_list.h \
p3filelists.h
HEADERS += chat/distantchat.h \
chat/p3chatservice.h \
chat/distributedchat.h \
@ -507,17 +527,9 @@ HEADERS += util/folderiterator.h \
util/rsscopetimer.h \
util/stacktrace.h
SOURCES += dbase/cachestrapper.cc \
dbase/fimonitor.cc \
dbase/findex.cc \
dbase/fistore.cc \
dbase/rsexpr.cc
SOURCES += ft/ftchunkmap.cc \
ft/ftcontroller.cc \
ft/ftdatamultiplex.cc \
ft/ftdbase.cc \
ft/ftextralist.cc \
ft/ftfilecreator.cc \
ft/ftfileprovider.cc \
@ -638,6 +650,7 @@ SOURCES += turtle/p3turtle.cc \
SOURCES += util/folderiterator.cc \
util/rsdebug.cc \
util/rsexpr.cc \
util/rscompress.cc \
util/smallobject.cc \
util/rsdir.cc \

16
libretroshare/src/plugins/pluginmanager.cc
View file

@ -17,7 +17,6 @@
#include <util/rsversioninfo.h>
#include <util/folderiterator.h>
#include <ft/ftserver.h>
#include <dbase/cachestrapper.h>
#include <retroshare/rsplugin.h>
#include <retroshare/rsfiles.h>
#include <pqi/pqiservice.h>
@ -146,7 +145,7 @@ void RsPluginManager::loadPlugins(const std::vector<std::string>& plugin_directo
continue ;
}
while(dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
std::string fname;
dirIt.d_name(fname);
@ -410,14 +409,17 @@ void RsPluginManager::slowTickPlugins(time_t seconds)
void RsPluginManager::registerCacheServices()
{
// this is removed since the old cache syste is gone, but we need to make it register new GXS group services instead.
#ifdef REMOVED
std::cerr << " Registering cache services." << std::endl;
for(uint32_t i=0;i<_plugins.size();++i)
if(_plugins[i].plugin != NULL && _plugins[i].plugin->rs_cache_service() != NULL)
{
rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ;
//rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ;
std::cerr << " adding new cache pair for plugin " << _plugins[i].plugin->getPluginName() << ", with RS_ID " << _plugins[i].plugin->rs_service_id() << std::endl ;
}
#endif
}
void RsPluginManager::registerClientServices(p3ServiceServer *pqih)
@ -546,14 +548,6 @@ bool RsPluginManager::saveList(bool& cleanup, std::list<RsItem*>& list)
return true;
}
// RsCacheService::RsCacheService(uint16_t service_type,uint32_t tick_delay, RsPluginHandler* pgHandler)
// : CacheSource(service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getLocalCacheDir()),
// CacheStore (service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getFileServer()->getCacheTransfer(), pgHandler->getRemoteCacheDir()),
// p3Config(),
// _tick_delay_in_seconds(tick_delay)
// {
// }
RsPQIService::RsPQIService(uint16_t /*service_type*/, uint32_t /*tick_delay_in_seconds*/, RsPluginHandler* /*pgHandler*/)
: p3Service(),p3Config()
{

3
libretroshare/src/plugins/rscacheservice.h
View file

@ -1,6 +1,5 @@
#pragma once
#include <dbase/cachestrapper.h>
#include "plugins/pluginmanager.h"
// This code needs to be re-written to work with GXS. For now it is obsolete.
@ -9,7 +8,7 @@
// The following class abstracts the construction of a cache service. The user only has to
// supply RS with a type ID. If the ID is already in use, RS will complain.
//
class RsCacheService: public CacheSource, public CacheStore, public p3Config
class RsCacheService: /* public CacheSource, public CacheStore, */ public p3Config
{
public:
RsCacheService(uint16_t type,uint32_t tick_delay_in_seconds, RsPluginHandler* pgHandler) ;

382
libretroshare/src/retroshare/rsexpr.h
View file

@ -1,6 +1,3 @@
#ifndef RS_EXPRESSIONS_H
#define RS_EXPRESSIONS_H
/*
* rs-core/src/rsiface: rsexpr.h
*
@ -26,38 +23,41 @@
*
*/
#pragma once
#include <string>
#include <list>
#include <stdint.h>
#include "retroshare/rstypes.h"
/******************************************************************************************
Enumerations defining the Operators usable in the Boolean search expressions
******************************************************************************************/
namespace RsRegularExpression
{
enum LogicalOperator{
AndOp=0, /* exp AND exp */
OrOp=1, /* exp OR exp */
XorOp=2 /* exp XOR exp */
AndOp=0, /* exp AND exp */
OrOp=1, /* exp OR exp */
XorOp=2 /* exp XOR exp */
};
/*Operators for String Queries*/
/* Operators for String Queries */
enum StringOperator{
ContainsAnyStrings = 0, /* e.g. name contains any of 'conference' 'meeting' 'presentation' */
ContainsAllStrings = 1, /* same as above except that it contains ALL of the strings */
EqualsString = 2 /* exactly equal*/
ContainsAnyStrings = 0, /* e.g. name contains any of 'conference' 'meeting' 'presentation' */
ContainsAllStrings = 1, /* same as above except that it contains ALL of the strings */
EqualsString = 2 /* exactly equal*/
};
/*Relational operators ( >, <, >=, <=, == and InRange )*/
/* Comparison operators ( >, <, >=, <=, == and InRange ) */
enum RelOperator{
Equals = 0,
GreaterEquals = 1,
Greater = 2,
SmallerEquals = 3,
Smaller = 4,
InRange = 5 /* lower limit <= value <= upper limit*/
Equals = 0,
GreaterEquals = 1,
Greater = 2,
SmallerEquals = 3,
Smaller = 4,
InRange = 5 /* lower limit <= value <= upper limit*/
};
/********************************************************************************************
@ -69,26 +69,26 @@ class Expression ;
class LinearizedExpression
{
public:
std::vector<uint8_t> _tokens ;
std::vector<uint32_t> _ints ;
std::vector<std::string> _strings ;
public:
std::vector<uint8_t> _tokens ;
std::vector<uint32_t> _ints ;
std::vector<std::string> _strings ;
typedef enum { EXPR_DATE= 0,
EXPR_POP = 1,
EXPR_SIZE= 2,
EXPR_HASH= 3,
EXPR_NAME= 4,
EXPR_PATH= 5,
EXPR_EXT = 6,
EXPR_COMP= 7,
EXPR_SIZE_MB=8 } token ;
typedef enum { EXPR_DATE = 0,
EXPR_POP = 1,
EXPR_SIZE = 2,
EXPR_HASH = 3,
EXPR_NAME = 4,
EXPR_PATH = 5,
EXPR_EXT = 6,
EXPR_COMP = 7,
EXPR_SIZE_MB = 8 } token ;
static Expression *toExpr(const LinearizedExpression& e) ;
static Expression *toExpr(const LinearizedExpression& e) ;
private:
static Expression *toExpr(const LinearizedExpression& e,int&,int&,int&) ;
static void readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list<std::string>& strings,bool& b,StringOperator& op) ;
private:
static Expression *toExpr(const LinearizedExpression& e,int&,int&,int&) ;
static void readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list<std::string>& strings,bool& b,StringOperator& op) ;
};
@ -96,110 +96,122 @@ class LinearizedExpression
Boolean Search Expression
classes:
Expression: The base class of all expression typest
CompoundExpression: The expression which uses a logical operator to combine
the results of two expressions
StringExpression: An expression which uses some sort of string comparison.
RelExpression: A Relational Expression where > < >= <= == make sense.
e.g. size date etc
Expression: The base class of all expression typest
CompoundExpression: The expression which uses a logical operator to combine
the results of two expressions
StringExpression: An expression which uses some sort of string comparison.
RelExpression: A Relational Expression where > < >= <= == make sense.
e.g. size date etc
******************************************************************************************/
class FileEntry;
/*!
* \brief The ExpFileEntry class
* This is the base class the regular expressions can operate on. Derive it to fit your own needs!
*/
class ExpFileEntry
{
public:
virtual const std::string& file_name() const =0;
virtual uint64_t file_size() const =0;
virtual time_t file_modtime() const =0;
virtual uint32_t file_popularity() const =0;
virtual std::string file_parent_path() const =0;
virtual const RsFileHash& file_hash() const =0;
};
class Expression
{
public:
virtual bool eval (FileEntry *file) = 0;
virtual ~Expression() {};
public:
virtual bool eval (const ExpFileEntry& file) = 0;
virtual ~Expression() {};
virtual void linearize(LinearizedExpression& e) const = 0 ;
virtual void linearize(LinearizedExpression& e) const = 0 ;
};
class CompoundExpression : public Expression
{
public:
CompoundExpression( enum LogicalOperator op, Expression * exp1, Expression *exp2)
: Lexp(exp1), Rexp(exp2), Op(op){ }
public:
CompoundExpression( enum LogicalOperator op, Expression * exp1, Expression *exp2)
: Lexp(exp1), Rexp(exp2), Op(op){ }
bool eval (FileEntry *file) {
if (Lexp == NULL or Rexp == NULL) {
return false;
}
switch (Op){
case AndOp:
return Lexp->eval(file) && Rexp->eval(file);
case OrOp:
return Lexp->eval(file) || Rexp->eval(file);
case XorOp:
return Lexp->eval(file) ^ Rexp->eval(file);
default:
return false;
}
}
virtual ~CompoundExpression(){
delete Lexp;
delete Rexp;
}
bool eval (const ExpFileEntry& file) {
if (Lexp == NULL or Rexp == NULL) {
return false;
}
switch (Op){
case AndOp:
return Lexp->eval(file) && Rexp->eval(file);
case OrOp:
return Lexp->eval(file) || Rexp->eval(file);
case XorOp:
return Lexp->eval(file) ^ Rexp->eval(file);
default:
return false;
}
}
virtual ~CompoundExpression(){
delete Lexp;
delete Rexp;
}
virtual void linearize(LinearizedExpression& e) const ;
private:
Expression *Lexp;
Expression *Rexp;
enum LogicalOperator Op;
virtual void linearize(LinearizedExpression& e) const ;
private:
Expression *Lexp;
Expression *Rexp;
enum LogicalOperator Op;
};
class StringExpression: public Expression
{
public:
StringExpression(enum StringOperator op, std::list<std::string> &t, bool ic): Op(op),terms(t), IgnoreCase(ic){}
public:
StringExpression(enum StringOperator op, std::list<std::string> &t, bool ic): Op(op),terms(t), IgnoreCase(ic){}
virtual void linearize(LinearizedExpression& e) const ;
protected:
bool evalStr(const std::string &str);
virtual void linearize(LinearizedExpression& e) const ;
protected:
bool evalStr(const std::string &str);
enum StringOperator Op;
std::list<std::string> terms;
bool IgnoreCase;
enum StringOperator Op;
std::list<std::string> terms;
bool IgnoreCase;
};
template <class T>
class RelExpression: public Expression
{
public:
RelExpression(enum RelOperator op, T lv, T hv): Op(op), LowerValue(lv), HigherValue(hv) {}
public:
RelExpression(enum RelOperator op, T lv, T hv): Op(op), LowerValue(lv), HigherValue(hv) {}
virtual void linearize(LinearizedExpression& e) const ;
protected:
bool evalRel(T val);
virtual void linearize(LinearizedExpression& e) const ;
protected:
bool evalRel(T val);
enum RelOperator Op;
T LowerValue;
T HigherValue;
enum RelOperator Op;
T LowerValue;
T HigherValue;
};
template<> void RelExpression<int>::linearize(LinearizedExpression& e) const ;
template <class T>
bool RelExpression<T>::evalRel(T val) {
switch (Op) {
case Equals:
return LowerValue == val;
case GreaterEquals:
return LowerValue >= val;
case Greater:
return LowerValue > val;
case SmallerEquals:
return LowerValue <= val;
case Smaller:
return LowerValue < val;
case InRange:
return (LowerValue <= val) && (val <= HigherValue);
default:
return false;
}
switch (Op) {
case Equals:
return LowerValue == val;
case GreaterEquals:
return LowerValue >= val;
case Greater:
return LowerValue > val;
case SmallerEquals:
return LowerValue <= val;
case Smaller:
return LowerValue < val;
case InRange:
return (LowerValue <= val) && (val <= HigherValue);
default:
return false;
}
}
@ -212,12 +224,12 @@ Binary Predicate for Case Insensitive search
*Factor locales in the comparison
*/
struct CompareCharIC :
public std::binary_function< char , char , bool> {
public std::binary_function< char , char , bool> {
bool operator () ( char ch1 , char ch2 ) const {
return tolower( static_cast < unsigned char > (ch1) )
== tolower( static_cast < unsigned char > (ch2) );
}
bool operator () ( char ch1 , char ch2 ) const {
return tolower( static_cast < unsigned char > (ch1) )
== tolower( static_cast < unsigned char > (ch2) );
}
};
@ -228,55 +240,55 @@ Some implementations of StringExpressions.
class NameExpression: public StringExpression
{
public:
NameExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(FileEntry *file);
public:
NameExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_NAME) ;
StringExpression::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_NAME) ;
StringExpression::linearize(e) ;
}
};
class PathExpression: public StringExpression {
public:
PathExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(FileEntry *file);
PathExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_PATH) ;
StringExpression::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_PATH) ;
StringExpression::linearize(e) ;
}
};
class ExtExpression: public StringExpression {
public:
ExtExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(FileEntry *file);
ExtExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_EXT) ;
StringExpression::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_EXT) ;
StringExpression::linearize(e) ;
}
};
class HashExpression: public StringExpression {
public:
HashExpression(enum StringOperator op, std::list<std::string> &t):
StringExpression(op,t, true) {}
bool eval(FileEntry *file);
HashExpression(enum StringOperator op, std::list<std::string> &t):
StringExpression(op,t, true) {}
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_HASH) ;
StringExpression::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_HASH) ;
StringExpression::linearize(e) ;
}
};
/******************************************************************************************
@ -286,62 +298,62 @@ Some implementations of Relational Expressions.
class DateExpression: public RelExpression<int>
{
public:
DateExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
DateExpression(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(FileEntry *file);
public:
DateExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
DateExpression(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_DATE) ;
RelExpression<int>::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_DATE) ;
RelExpression<int>::linearize(e) ;
}
};
class SizeExpression: public RelExpression<int>
{
public:
SizeExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
SizeExpression(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(FileEntry *file);
public:
SizeExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
SizeExpression(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_SIZE) ;
RelExpression<int>::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_SIZE) ;
RelExpression<int>::linearize(e) ;
}
};
class SizeExpressionMB: public RelExpression<int>
{
public:
SizeExpressionMB(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
SizeExpressionMB(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(FileEntry *file);
public:
SizeExpressionMB(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
SizeExpressionMB(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_SIZE_MB) ;
RelExpression<int>::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_SIZE_MB) ;
RelExpression<int>::linearize(e) ;
}
};
class PopExpression: public RelExpression<int>
{
public:
PopExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
PopExpression(enum RelOperator op, int lv, int hv): RelExpression<int>(op,lv,hv) {}
PopExpression(const LinearizedExpression& e) ;
bool eval(FileEntry *file);
public:
PopExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
PopExpression(enum RelOperator op, int lv, int hv): RelExpression<int>(op,lv,hv) {}
PopExpression(const LinearizedExpression& e) ;
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_POP) ;
RelExpression<int>::linearize(e) ;
}
virtual void linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_POP) ;
RelExpression<int>::linearize(e) ;
}
};
}
#endif /* RS_EXPRESSIONS_H */

50
libretroshare/src/retroshare/rsfiles.h
View file

@ -36,9 +36,7 @@
class RsFiles;
extern RsFiles *rsFiles;
class Expression;
class CacheStrapper ;
class CacheTransfer;
namespace RsRegularExpression { class Expression; }
/* These are used mainly by ftController at the moment */
const uint32_t RS_FILE_CTRL_PAUSE = 0x00000100;
@ -65,26 +63,26 @@ const uint32_t RS_FILE_PEER_OFFLINE = 0x00002000;
// Flags used when requesting info about transfers, mostly to filter out the result.
//
const FileSearchFlags RS_FILE_HINTS_CACHE ( 0x00000001 );
const FileSearchFlags RS_FILE_HINTS_EXTRA ( 0x00000002 );
const FileSearchFlags RS_FILE_HINTS_LOCAL ( 0x00000004 );
const FileSearchFlags RS_FILE_HINTS_REMOTE ( 0x00000008 );
const FileSearchFlags RS_FILE_HINTS_CACHE_deprecated ( 0x00000001 );
const FileSearchFlags RS_FILE_HINTS_EXTRA ( 0x00000002 );
const FileSearchFlags RS_FILE_HINTS_LOCAL ( 0x00000004 );
const FileSearchFlags RS_FILE_HINTS_REMOTE ( 0x00000008 );
const FileSearchFlags RS_FILE_HINTS_DOWNLOAD ( 0x00000010 );
const FileSearchFlags RS_FILE_HINTS_UPLOAD ( 0x00000020 );
const FileSearchFlags RS_FILE_HINTS_SPEC_ONLY ( 0x01000000 );
const FileSearchFlags RS_FILE_HINTS_UPLOAD ( 0x00000020 );
const FileSearchFlags RS_FILE_HINTS_SPEC_ONLY ( 0x01000000 );
const FileSearchFlags RS_FILE_HINTS_NETWORK_WIDE ( 0x00000080 );// anonymously shared over network
const FileSearchFlags RS_FILE_HINTS_BROWSABLE ( 0x00000100 );// browsable by friends
const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000180 );// OR of the last two flags. Used to filter out.
const FileSearchFlags RS_FILE_HINTS_NETWORK_WIDE ( 0x00000080 );// anonymously shared over network
const FileSearchFlags RS_FILE_HINTS_BROWSABLE ( 0x00000100 );// browsable by friends
const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000180 );// OR of the last two flags. Used to filter out.
// Flags used when requesting a transfer
//
const TransferRequestFlags RS_FILE_REQ_ANONYMOUS_ROUTING ( 0x00000040 ); // Use to ask turtle router to download the file.
const TransferRequestFlags RS_FILE_REQ_ASSUME_AVAILABILITY ( 0x00000200 ); // Assume full source availability. Used for cache files.
const TransferRequestFlags RS_FILE_REQ_CACHE ( 0x00000400 ); // Assume full source availability. Used for cache files.
const TransferRequestFlags RS_FILE_REQ_CACHE_deprecated ( 0x00000400 ); // Assume full source availability. Used for cache files.
const TransferRequestFlags RS_FILE_REQ_EXTRA ( 0x00000800 );
const TransferRequestFlags RS_FILE_REQ_MEDIA ( 0x00001000 );
const TransferRequestFlags RS_FILE_REQ_BACKGROUND ( 0x00002000 ); // To download slowly.
const TransferRequestFlags RS_FILE_REQ_BACKGROUND ( 0x00002000 ); // To download slowly.
const TransferRequestFlags RS_FILE_REQ_NO_SEARCH ( 0x02000000 ); // disable searching for potential direct sources.
// const uint32_t RS_FILE_HINTS_SHARE_FLAGS_MASK = RS_FILE_HINTS_NETWORK_WIDE_OTHERS | RS_FILE_HINTS_BROWSABLE_OTHERS
@ -147,8 +145,6 @@ class RsFiles
/***
* Control of Downloads Priority.
***/
virtual uint32_t getMinPrioritizedTransfers() = 0 ;
virtual void setMinPrioritizedTransfers(uint32_t s) = 0 ;
virtual uint32_t getQueueSize() = 0 ;
virtual void setQueueSize(uint32_t s) = 0 ;
virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove mv) = 0;
@ -186,12 +182,13 @@ class RsFiles
*/
virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) = 0;
virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) = 0;
virtual uint32_t getType(void *ref,FileSearchFlags flags) = 0;
virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) =0;
virtual uint32_t getType(void *ref,FileSearchFlags flags) = 0;
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags) = 0;
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0;
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags) = 0;
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0;
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags) = 0;
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0;
/***
* Utility Functions.
@ -205,6 +202,8 @@ class RsFiles
/***
* Directory Control
***/
virtual void requestDirUpdate(void *ref) =0 ; // triggers the update of the given reference. Used when browsing.
virtual void setDownloadDirectory(std::string path) = 0;
virtual void setPartialsDirectory(std::string path) = 0;
virtual std::string getDownloadDirectory() = 0;
@ -214,16 +213,11 @@ class RsFiles
virtual bool addSharedDirectory(const SharedDirInfo& dir) = 0;
virtual bool updateShareFlags(const SharedDirInfo& dir) = 0; // updates the flags. The directory should already exist !
virtual bool removeSharedDirectory(std::string dir) = 0;
virtual void setRememberHashFilesDuration(uint32_t days) = 0 ;
virtual uint32_t rememberHashFilesDuration() const = 0 ;
virtual void clearHashCache() = 0 ;
virtual bool rememberHashFiles() const =0;
virtual void setRememberHashFiles(bool) =0;
virtual void setWatchPeriod(int minutes) =0;
virtual int watchPeriod() const =0;
virtual CacheStrapper *getCacheStrapper() =0;
virtual CacheTransfer *getCacheTransfer() =0;
virtual void setWatchPeriod(int minutes) =0;
virtual void setWatchEnabled(bool b) =0;
virtual int watchPeriod() const =0;
virtual bool watchEnabled() =0;
virtual bool getShareDownloadDirectory() = 0;
virtual bool shareDownloadDirectory(bool share) = 0;

4
libretroshare/src/retroshare/rsturtle.h
View file

@ -35,7 +35,7 @@
#include "retroshare/rstypes.h"
class LinearizedExpression ;
namespace RsRegularExpression { class LinearizedExpression ; }
class RsTurtleClientService ;
class RsTurtle;
@ -102,7 +102,7 @@ class RsTurtle
// as they come back.
//
virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ;
virtual TurtleRequestId turtleSearch(const LinearizedExpression& expr) = 0 ;
virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ;
// Initiates tunnel handling for the given file hash. tunnels. Launches
// an exception if an error occurs during the initialization process. The

24
libretroshare/src/retroshare/rstypes.h
View file

@ -138,6 +138,7 @@ public:
/********************** For FileCache Interface *****************/
#define DIR_TYPE_UNKNOWN 0x00
#define DIR_TYPE_ROOT 0x01
#define DIR_TYPE_PERSON 0x02
#define DIR_TYPE_DIR 0x04
@ -159,7 +160,7 @@ const FileStorageFlags DIR_FLAGS_BROWSABLE_OTHERS ( 0x0100 ); // one should
const FileStorageFlags DIR_FLAGS_NETWORK_WIDE_GROUPS ( 0x0200 );
const FileStorageFlags DIR_FLAGS_BROWSABLE_GROUPS ( 0x0400 );
const FileStorageFlags DIR_FLAGS_PERMISSIONS_MASK ( DIR_FLAGS_NETWORK_WIDE_OTHERS | DIR_FLAGS_BROWSABLE_OTHERS
| DIR_FLAGS_NETWORK_WIDE_GROUPS | DIR_FLAGS_BROWSABLE_GROUPS );
| DIR_FLAGS_NETWORK_WIDE_GROUPS | DIR_FLAGS_BROWSABLE_GROUPS );
const FileStorageFlags DIR_FLAGS_LOCAL ( 0x1000 );
const FileStorageFlags DIR_FLAGS_REMOTE ( 0x2000 );
@ -214,8 +215,7 @@ class FileInfo
std::list<RsNodeGroupId> parent_groups ;
};
std::ostream &operator<<(std::ostream &out, const FileInfo &info);
std::ostream &operator<<(std::ostream &out, const FileInfo& info);
class DirStub
{
@ -227,25 +227,27 @@ class DirStub
class DirDetails
{
public:
void *parent;
int prow; /* parent row */
public:
void *parent;
int prow; /* parent row */
void *ref;
uint8_t type;
void *ref;
uint8_t type;
RsPeerId id;
std::string name;
std::string name;
RsFileHash hash;
std::string path;
std::string path; // full path of the parent directory, when it is a file; full path of the dir otherwise.
uint64_t count;
uint32_t age;
FileStorageFlags flags;
uint32_t min_age ; // minimum age of files in this subtree
std::list<DirStub> children;
std::vector<DirStub> children;
std::list<RsNodeGroupId> parent_groups; // parent groups for the shared directory
};
std::ostream &operator<<(std::ostream &out, const DirDetails& details);
class FileDetail
{
public:

2
libretroshare/src/rsserver/rsaccounts.cc
View file

@ -533,7 +533,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map<RsPeerId, AccountDetails> &
struct stat64 buf;
while (dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
/* check entry type */
std::string fname;

32
libretroshare/src/rsserver/rsinit.cc
View file

@ -56,7 +56,8 @@
#include <openssl/rand.h>
#include <fcntl.h>
#include <gxstunnel/p3gxstunnel.h>
#include "gxstunnel/p3gxstunnel.h"
#include "file_sharing/p3filelists.h"
#define ENABLE_GROUTER
@ -804,7 +805,6 @@ bool RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port)
#include <unistd.h>
//#include <getopt.h>
#include "dbase/cachestrapper.h"
#include "ft/ftserver.h"
#include "ft/ftcontroller.h"
@ -1239,12 +1239,10 @@ int RsServer::StartupRetroShare()
//pqih = new pqipersongrpDummy(none, flags);
/****** New Ft Server **** !!! */
ftServer *ftserver = new ftServer(mPeerMgr, serviceCtrl);
ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory());
ftServer *ftserver = new ftServer(mPeerMgr, serviceCtrl);
ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory());
ftserver->SetupFtServer() ;
CacheStrapper *mCacheStrapper = ftserver->getCacheStrapper();
//CacheTransfer *mCacheTransfer = ftserver->getCacheTransfer();
/* setup any extra bits (Default Paths) */
ftserver->setPartialsDirectory(emergencyPartialsDir);
@ -1288,7 +1286,6 @@ int RsServer::StartupRetroShare()
// These are needed to load plugins: plugin devs might want to know the place of
// cache directories, get pointers to cache strapper, or access ownId()
//
mPluginsManager->setCacheDirectories(localcachedir,remotecachedir) ;
mPluginsManager->setServiceControl(serviceCtrl) ;
// std::cerr << "rsinitconf (core 1) = " << (void*)rsInitConfig<<std::endl;
@ -1499,22 +1496,25 @@ int RsServer::StartupRetroShare()
pqih->addService(gr,true) ;
#endif
p3turtle *tr = new p3turtle(serviceCtrl,mLinkMgr) ;
p3FileDatabase *fdb = new p3FileDatabase(serviceCtrl) ;
p3turtle *tr = new p3turtle(serviceCtrl,mLinkMgr) ;
rsTurtle = tr ;
pqih -> addService(tr,true);
pqih -> addService(ftserver,true);
pqih -> addService(fdb,true);
pqih -> addService(ftserver,true);
mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ;
mGxsTunnels->connectToTurtleRouter(tr) ;
rsGxsTunnel = mGxsTunnels;
rsDisc = mDisc;
rsMsgs = new p3Msgs(msgSrv, chatSrv);
// connect components to turtle router.
ftserver->connectToTurtleRouter(tr) ;
chatSrv->connectToGxsTunnelService(mGxsTunnels) ;
ftserver->connectToFileDatabase(fdb) ;
chatSrv->connectToGxsTunnelService(mGxsTunnels) ;
gr->connectToTurtleRouter(tr) ;
#ifdef ENABLE_GROUTER
msgSrv->connectToGlobalRouter(gr) ;
@ -1619,10 +1619,6 @@ int RsServer::StartupRetroShare()
mLinkMgr->addMonitor(serviceCtrl);
mLinkMgr->addMonitor(serviceInfo);
// NOTE these were added in ftServer (was added twice).
//mLinkMgr->addMonitor(mCacheStrapper);
//mLinkMgr->addMonitor(((ftController *) mCacheTransfer));
// Services that have been changed to pqiServiceMonitor
serviceCtrl->registerServiceMonitor(msgSrv, msgSrv->getServiceInfo().mServiceType);
serviceCtrl->registerServiceMonitor(mDisc, mDisc->getServiceInfo().mServiceType);
@ -1678,12 +1674,6 @@ int RsServer::StartupRetroShare()
/**************************************************************************/
std::cerr << "(2) Load configuration files" << std::endl;
/* NOTE: CacheStrapper's load causes Cache Files to be
* loaded into all the CacheStores/Sources. This happens
* after all the other configurations have happened.
*/
mConfigMgr->addConfiguration("cache.cfg", mCacheStrapper);
mConfigMgr->loadConfiguration();
/**************************************************************************/

21
libretroshare/src/rsserver/rstypes.cc
View file

@ -44,7 +44,26 @@
*
**********************************************************************/
std::ostream &operator<<(std::ostream &out, const DirDetails& d)
{
std::cerr << "====DIR DETAILS====" << std::endl;
std::cerr << " parent pointer: " << d.parent << std::endl;
std::cerr << " current pointer: " << d.ref << std::endl;
std::cerr << " parent row : " << d.prow << std::endl;
std::cerr << " type : " << (int)d.type << std::endl;
std::cerr << " PeerId : " << d.id << std::endl;
std::cerr << " Name : " << d.name << std::endl;
std::cerr << " Hash : " << d.hash << std::endl;
std::cerr << " Path : " << d.path << std::endl;
std::cerr << " Count : " << d.count << std::endl;
std::cerr << " Age : " << d.age << std::endl;
std::cerr << " Min age : " << d.min_age << std::endl;
std::cerr << " Flags : " << d.flags << std::endl;
std::cerr << " Parent groups : " ; for(std::list<RsNodeGroupId>::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl;
std::cerr << " Children : " ; for(uint32_t i=0;i<d.children.size();++i) std::cerr << (void*)(intptr_t)d.children[i].ref << " "; std::cerr << std::endl;
std::cerr << "===================" << std::endl;
return out;
}
std::ostream &operator<<(std::ostream &out, const FileInfo &info)
{
out << "FileInfo: path: " << info.path;

5
libretroshare/src/serialiser/itempriorities.h
View file

@ -76,6 +76,11 @@ const uint8_t QOS_PRIORITY_RS_DISC_PGP_LIST = 2 ; // same priority.
const uint8_t QOS_PRIORITY_RS_DISC_SERVICES = 2 ;
const uint8_t QOS_PRIORITY_RS_DISC_PGP_CERT = 1 ;
// File database
//
const uint8_t QOS_PRIORITY_RS_FAST_SYNC_REQUEST = 7 ;
const uint8_t QOS_PRIORITY_RS_SLOW_SYNC_REQUEST = 3 ;
// Heartbeat.
//
const uint8_t QOS_PRIORITY_RS_HEARTBEAT_PULSE = 8 ;

3
libretroshare/src/serialiser/rsbaseserial.cc
View file

@ -231,6 +231,9 @@ uint32_t getRawStringSize(const std::string &outStr)
bool getRawString(void *data, uint32_t size, uint32_t *offset, std::string &outStr)
{
#warning I had to change this. It seems like a bug to not clear the string. Should make sure it's not introducing any side effect.
outStr.clear();
uint32_t len = 0;
if (!getRawUInt32(data, size, offset, &len))
{

1
libretroshare/src/serialiser/rsserviceids.h
View file

@ -49,6 +49,7 @@ const uint16_t RS_SERVICE_TYPE_TUNNEL = 0x0015;
const uint16_t RS_SERVICE_TYPE_HEARTBEAT = 0x0016;
const uint16_t RS_SERVICE_TYPE_FILE_TRANSFER = 0x0017;
const uint16_t RS_SERVICE_TYPE_GROUTER = 0x0018;
const uint16_t RS_SERVICE_TYPE_FILE_DATABASE = 0x0019;
const uint16_t RS_SERVICE_TYPE_SERVICEINFO = 0x0020;
/* Bandwidth Control */

4
libretroshare/src/turtle/p3turtle.cc
View file

@ -1763,7 +1763,7 @@ void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
std::list<DirDetails> initialResults;
// to do: split search string into words.
Expression *exp = LinearizedExpression::toExpr(expr) ;
RsRegularExpression::Expression *exp = RsRegularExpression::LinearizedExpression::toExpr(expr) ;
if(exp == NULL)
return ;
@ -1820,7 +1820,7 @@ TurtleRequestId p3turtle::turtleSearch(const std::string& string_to_match)
return id ;
}
TurtleRequestId p3turtle::turtleSearch(const LinearizedExpression& expr)
TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpression& expr)
{
// generate a new search id.

2
libretroshare/src/turtle/p3turtle.h
View file

@ -239,7 +239,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
// remove the specific file search packets from the turtle router.
//
virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
virtual TurtleSearchRequestId turtleSearch(const LinearizedExpression& expr) ;
virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
// Initiates tunnel handling for the given file hash. tunnels. Launches
// an exception if an error occurs during the initialization process. The

2
libretroshare/src/turtle/rsturtleitem.h
View file

@ -102,7 +102,7 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem
RsTurtleRegExpSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {}
RsTurtleRegExpSearchRequestItem(void *data,uint32_t size) ;
LinearizedExpression expr ; // Reg Exp in linearised mode
RsRegularExpression::LinearizedExpression expr ; // Reg Exp in linearised mode
virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; }
virtual void performLocalSearch(std::list<TurtleFileInfo>&) const ;

115
libretroshare/src/util/folderiterator.cc
View file

@ -1,12 +1,39 @@
#include <dirent.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#ifdef WINDOWS_SYS
#include "util/rswin.h"
#endif
#include "folderiterator.h"
#include "rsstring.h"
namespace librs { namespace util {
FolderIterator::FolderIterator(const std::string& folderName)
: mFolderName(folderName)
{
// Grab the last modification time for the directory
struct stat64 buf ;
#ifdef WINDOWS_SYS
std::wstring wfullname;
librs::util::ConvertUtf8ToUtf16(folderName, wfullname);
if ( 0 == _wstati64(wfullname.c_str(), &buf))
#else
if ( 0 == stat64(folderName.c_str(), &buf))
#endif
{
mFolderModTime = buf.st_mtime ;
}
// Now open directory content and read the first entry
#ifdef WINDOWS_SYS
std::wstring utf16Name;
if(! ConvertUtf8ToUtf16(folderName, utf16Name)) {
@ -17,11 +44,11 @@ FolderIterator::FolderIterator(const std::string& folderName)
utf16Name += L"/*.*";
handle = FindFirstFileW(utf16Name.c_str(), &fileInfo);
validity = handle != INVALID_HANDLE_VALUE;
isFirstCall = true;
is_open = validity = handle != INVALID_HANDLE_VALUE;
#else
handle = opendir(folderName.c_str());
validity = handle != NULL;
is_open = validity = handle != NULL;
next();
#endif
}
@ -30,19 +57,71 @@ FolderIterator::~FolderIterator()
closedir();
}
bool FolderIterator::readdir() {
void FolderIterator::next()
{
do {
if(!readdir())
{
validity = false ;
break ;
}
d_name(mFileName);
} while(mFileName == "." || mFileName == "..") ;
mFullPath = mFolderName + "/" + mFileName ;
struct stat64 buf ;
#ifdef WINDOWS_SYS
std::wstring wfullname;
librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname);
if ( 0 == _wstati64(wfullname.c_str(), &buf))
#else
if ( 0 == stat64(mFullPath.c_str(), &buf))
#endif
{
mFileModTime = buf.st_mtime ;
mStatInfoOk = true;
if (S_ISDIR(buf.st_mode))
{
mType = TYPE_DIR ;
mFileSize = 0 ;
mFileModTime = buf.st_mtime;
}
else if (S_ISREG(buf.st_mode))
{
mType = TYPE_FILE ;
mFileSize = buf.st_size;
mFileModTime = buf.st_mtime;
}
else
{
mType = TYPE_UNKNOWN ;
mFileSize = 0 ;
mFileModTime = 0;
}
}
else
{
mType = TYPE_UNKNOWN ;
mFileSize = 0 ;
mFileModTime = 0;
validity = false ;
}
}
bool FolderIterator::readdir()
{
if(!validity)
return false;
#ifdef WINDOWS_SYS
if(isFirstCall) {
isFirstCall = false;
return true;
}
return FindNextFileW(handle, &fileInfo) != 0;
#else
ent = ::readdir(handle);
return ent != 0;
return ent != NULL;
#endif
}
@ -65,13 +144,23 @@ bool FolderIterator::d_name(std::string& dest)
return true;
}
time_t FolderIterator::dir_modtime() const { return mFolderModTime ; }
const std::string& FolderIterator::file_fullpath() { return mFullPath ; }
const std::string& FolderIterator::file_name() { return mFileName ; }
uint64_t FolderIterator::file_size() { return mFileSize ; }
time_t FolderIterator::file_modtime() { return mFileModTime ; }
uint8_t FolderIterator::file_type() { return mType ; }
bool FolderIterator::closedir()
{
if(!validity)
return false;
validity = false;
if(!is_open)
return true ;
is_open = false ;
#ifdef WINDOWS_SYS
return FindClose(handle) != 0;
#else

31
libretroshare/src/util/folderiterator.h
View file

@ -2,6 +2,7 @@
#define FOLDERITERATOR_H
#include <stdint.h>
#include <iostream>
#include <cstdio>
@ -24,15 +25,32 @@ public:
FolderIterator(const std::string& folderName);
~FolderIterator();
enum { TYPE_UNKNOWN = 0x00,
TYPE_FILE = 0x01,
TYPE_DIR = 0x02
};
// info about current parent directory
time_t dir_modtime() const ;
// info about directory content
bool isValid() const { return validity; }
bool readdir();
void next();
#warning this one should go, as it reports the same information than file_name()
bool d_name(std::string& dest);
bool closedir();
const std::string& file_name() ;
const std::string& file_fullpath() ;
uint64_t file_size() ;
uint8_t file_type() ;
time_t file_modtime() ;
private:
bool is_open;
bool validity;
#ifdef WINDOWS_SYS
@ -43,7 +61,16 @@ private:
DIR* handle;
struct dirent* ent;
#endif
void updateStatsInfo() ;
bool mStatInfoOk ;
time_t mFileModTime ;
time_t mFolderModTime ;
uint64_t mFileSize ;
uint8_t mType ;
std::string mFileName ;
std::string mFullPath ;
std::string mFolderName ;
};

287
libretroshare/src/util/rsexpr.cc Normal file
View file

@ -0,0 +1,287 @@
/*
* rs-core/src/dbase: rsexpr.cc
*
* RetroShare C++ Interface.
*
* Copyright 2007-2008 by Kashif Kaleem.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "retroshare/rsexpr.h"
#include "retroshare/rstypes.h"
#include <algorithm>
#include <functional>
/******************************************************************************************
eval functions of relational expressions.
******************************************************************************************/
namespace RsRegularExpression
{
template<>
void RelExpression<int>::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(LowerValue) ;
e._ints.push_back(HigherValue) ;
}
bool DateExpression::eval(const ExpFileEntry& file)
{
return evalRel(file.file_modtime());
}
bool SizeExpressionMB::eval(const ExpFileEntry& file)
{
return evalRel((int)(file.file_size()/(uint64_t)(1024*1024)));
}
bool SizeExpression::eval(const ExpFileEntry& file)
{
return evalRel(file.file_size());
}
bool PopExpression::eval(const ExpFileEntry& file)
{
return evalRel(file.file_popularity());
}
/******************************************************************************************
Code for evaluating string expressions
******************************************************************************************/
bool NameExpression::eval(const ExpFileEntry& file)
{
return evalStr(file.file_name());
}
bool PathExpression::eval(const ExpFileEntry& file)
{
return evalStr(file.file_parent_path());
}
bool ExtExpression::eval(const ExpFileEntry& file)
{
std::string ext;
/*Get the part of the string after the last instance of . in the filename */
size_t index = file.file_name().find_last_of('.');
if (index != std::string::npos) {
ext = file.file_name().substr(index+1);
if (ext != "" ){
return evalStr(ext);
}
}
return false;
}
bool HashExpression::eval(const ExpFileEntry& file){
return evalStr(file.file_hash().toStdString());
}
/*Check whether two strings are 'equal' to each other*/
static bool StrEquals(const std::string & str1, const std::string & str2,
bool IgnoreCase ){
if ( str1.size() != str2.size() ){
return false;
} else if (IgnoreCase) {
std::equal( str1.begin(), str1.end(),
str2.begin(), CompareCharIC() );
}
return std::equal( str1.begin(), str1.end(),
str2.begin());
}
/*Check whether one string contains the other*/
static bool StrContains( const std::string & str1, const std::string & str2,
bool IgnoreCase){
std::string::const_iterator iter ;
if (IgnoreCase) {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end(), CompareCharIC() );
} else {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end());
}
return ( iter != str1.end() );
}
bool StringExpression :: evalStr ( const std::string &str ){
std::list<std::string>::iterator iter;
switch (Op) {
case ContainsAllStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str, *iter, IgnoreCase) == false ){
return false;
}
}
return true;
break;
case ContainsAnyStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
case EqualsString:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrEquals (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
default:
return false;
}
return false;
}
/*************************************************************************
* linearization code
*************************************************************************/
void CompoundExpression::linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_COMP) ;
e._ints.push_back(Op) ;
Lexp->linearize(e) ;
Rexp->linearize(e) ;
}
void StringExpression::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(IgnoreCase) ;
e._ints.push_back(terms.size()) ;
for(std::list<std::string>::const_iterator it(terms.begin());it!=terms.end();++it)
e._strings.push_back(*it) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e)
{
int i=0,j=0,k=0 ;
return toExpr(e,i,j,k) ;
}
void LinearizedExpression::readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list<std::string>& strings,bool& b,StringOperator& op)
{
op = static_cast<StringOperator>(e._ints[n_ints++]) ;
b = e._ints[n_ints++] ;
int n = e._ints[n_ints++] ;
strings.clear() ;
for(int i=0;i<n;++i)
strings.push_back(e._strings[n_strings++]) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e,int& n_tok,int& n_ints,int& n_strings)
{
LinearizedExpression::token tok = static_cast<LinearizedExpression::token>(e._tokens[n_tok++]) ;
switch(tok)
{
case EXPR_DATE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new DateExpression(op,lv,hv) ;
}
case EXPR_POP: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new PopExpression(op,lv,hv) ;
}
case EXPR_SIZE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpression(op,lv,hv) ;
}
case EXPR_HASH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new HashExpression(op,strings) ;
}
case EXPR_NAME: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new NameExpression(op,strings,b) ;
}
case EXPR_PATH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_EXT: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_COMP: {
LogicalOperator op = static_cast<LogicalOperator>(e._ints[n_ints++]) ;
Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ;
Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ;
return new CompoundExpression(op,e1,e2) ;
}
case EXPR_SIZE_MB: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpressionMB(op,lv,hv) ;
}
default:
std::cerr << "No expression match the current value " << tok << std::endl ;
return NULL ;
}
}
}