Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

merged branch v0.6-FileLists locally into master

Browse Source
This commit is contained in:
csoler 2016-09-24 14:47:45 +02:00
commit ee54c02101
80 changed files with 7400 additions and 7939 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -6,6 +6,7 @@
moc_*.cpp
qrc_*.cpp
ui_*.h
Makefile.*
Makefile*
*.sw?
*~
Thumbs.db

4
libresapi/src/api/FileSearchHandler.cpp
View File

@ -173,8 +173,8 @@ void FileSearchHandler::handleCreateSearch(Request &req, Response &resp)
return;
}
NameExpression exprs(ContainsAllStrings,words,true) ;
LinearizedExpression lin_exp ;
RsRegularExpression::NameExpression exprs(RsRegularExpression::ContainsAllStrings,words,true) ;
RsRegularExpression::LinearizedExpression lin_exp ;
exprs.linearize(lin_exp) ;
uint32_t search_id = RSRandom::random_u32();

1323
libretroshare/src/dbase/cachestrapper.cc
View File

File diff suppressed because it is too large Load Diff

475
libretroshare/src/dbase/cachestrapper.h
View File

@ -1,475 +0,0 @@
/*
* RetroShare FileCache Module: cachestrapper.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_CACHE_STRAPPER_H
#define MRK_CACHE_STRAPPER_H
#include "pqi/p3cfgmgr.h"
#include "pqi/pqiservicemonitor.h"
#include "util/rsthreads.h"
#include <list>
#include <set>
#include <map>
#include <string>
#include <iostream>
/******************* CacheStrapper and Related Classes *******************
* A generic Cache Update system.
*
* CacheStrapper: maintains a set of CacheSources, and CacheStores,
* queries and updates as new information arrives.
*
* CacheTransfer: Interface for FileTransfer Class to support.
*
* CacheSource: Base Class for cache data provider. eg. FileIndexMonitor.
* CacheStore: Base Class for data cache. eg. FileCache/Store.
*
* Still TODO:
* (1) Design and Implement the Upload side of CacheTransfer/CacheStrapper.
* (2) CacheStrapper:: Save / Load Cache lists....
* (3) Clean up lists, maps on shutdown etc.
* (4) Consider Mutexes for multithreaded operations.
* (5) Test the MultiSource/Store capabilities.
*
******************* CacheStrapper and Related Classes *******************/
class CacheTransfer; /* Interface for File Transfer */
class CacheSource; /* Interface for local File Index/Monitor */
class CacheStore; /* Interface for the actual Cache */
class CacheStrapper; /* Controlling Class */
/******************************** CacheId ********************************/
/*!
* Use this to identify the type of cache source, strapper,
*/
class CacheId
{
public:
CacheId() :type(0), subid(0) { return; }
CacheId(uint16_t a, uint16_t b) :type(a), subid(b) { return; }
uint16_t type; /// cache types, this should be set to services type
uint16_t subid; /// should be initialised when using multicache feature of cachestrapper
};
bool operator<(const CacheId &a, const CacheId &b);
/*!
* Use for identifying physical files that have been chosen as cache data
* note: this does not actual store the data but serves to locate on network (via hash attribute,
* and on file via path)
*/
class RsCacheData
{
public:
RsPeerId pid; /// peer id
/// REMOVED as a WASTE to look it up everywhere! std::string pname; /// peer name (can be used by cachestore)
CacheId cid; /// cache id
std::string path; /// file system path where physical cache data is located
std::string name;
RsFileHash hash;
uint64_t size;
time_t recvd; /// received timestamp
};
std::ostream &operator<<(std::ostream &out, const RsCacheData &d);
/***************************** CacheTransfer *****************************/
/**
* Interface for FileTransfer Class to support cache
*/
class CacheTransfer
{
public:
CacheTransfer(CacheStrapper *cs) :strapper(cs) { return; }
virtual ~CacheTransfer() {}
/*!
* upload side of things .... searches through CacheStrapper.
*/
bool FindCacheFile(const RsFileHash& hash, std::string &path, uint64_t &size);
/*!
* At the download side RequestCache() => overloaded RequestCacheFile()
* the class should then call CompletedCache() or FailedCache()
*/
bool RequestCache(RsCacheData &data, CacheStore *cbStore); /* request from CacheStore */
protected:
/*!
* to be overloaded
*/
virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
bool CompletedCache(const RsFileHash &hash); /* internal completion -> does cb */
bool FailedCache(const RsFileHash &hash); /* internal completion -> does cb */
private:
CacheStrapper *strapper;
std::map<RsFileHash, RsCacheData> cbData;
std::map<RsFileHash, CacheStore *> cbStores;
};
/************************ CacheSource/CacheStore *************************/
typedef std::map<uint16_t, RsCacheData> CacheSet;
/*!
* Implements features needed for a service to act as a cachesource and allow pushing a of cache data from service to strapper
* Service is able to use this class for refresh its cache (push cache data)
* and interface to load and check cache availablility among peers (source of cache data)
* Architecturally Cachestrapper maintains the cachesource (which is passed as a pointer handle) while the cachesource-inheriting
* service can update cachesource as to new cache sources (cache data) created. Equivalently it enquiries through cache source for
* new cache data from peers
* @see p3Distrib
*/
class CacheSource
{
public:
CacheSource(uint16_t t, bool m, CacheStrapper *cs, std::string cachedir);
virtual ~CacheSource() {}
/*!
* called to determine available cache for peer -
* default acceptable (returns all)
*/
virtual bool cachesAvailable(const RsPeerId& pid, std::map<CacheId, RsCacheData> &ids)=0;
/*!
* function called at startup to load from
* configuration file....
* to be overloaded by inherited class
*/
virtual bool loadLocalCache(const RsCacheData &data);
/* control Caches available */
bool refreshCache(const RsCacheData &data,const std::set<RsPeerId>& destination_peers);
bool refreshCache(const RsCacheData &data);
bool clearCache(CacheId id);
/* controls if peer is an accepted receiver for cache items. Default is yes. To be overloaded. */
virtual bool isPeerAcceptedAsCacheReceiver(const RsPeerId& /*peer_id*/) { return true ; }
/* get private data */
std::string getCacheDir() { return cacheDir; }
bool isMultiCache() { return multiCache; }
uint16_t getCacheType() { return cacheType; }
/* display */
void listCaches(std::ostream &out);
/* search */
bool findCache(const RsFileHash& hash, RsCacheData &data) const;
protected:
uint16_t cacheType; /// for checking of cache type (usually of child class of source)
bool multiCache; /// whether multisource is in use or not.
CacheStrapper *mStrapper;
/*** MUTEX LOCKING */
void lockData() const;
void unlockData() const;
CacheSet caches; /// all local cache data stored here
std::map<RsFileHash, RsCacheData> mOldCaches; /// replaced/cleared caches are pushed here (in case requested)
private:
std::string cacheDir;
mutable RsMutex cMutex;
};
/*!
* Base Class for data cache. eg. FileCache/Store.
* This is best used to deal with external caches from other peers
* @see p3Distrib. pqiMonitor
*/
class CacheStore
{
public:
/*!
*
* @param t set to particular rs_service id. see rsserviceids.h
* @param m whether this is multicache service (true) or not(false)
* @param cs cache strapper instance responsible for maintaining the cache service
* @param cft cache transfer instance responsible for rquestiing and tranfering caches
* @param cachedir directory used to store cache related info for cachestore client
* @return
*/
CacheStore(uint16_t t, bool m, CacheStrapper *cs, CacheTransfer *cft, std::string cachedir);
virtual ~CacheStore() {}
/* current stored data */
/*!
*
* @param data returns cache data for pid/cid set in data itself
* @return false is unsuccessful and vice versa
*/
bool getStoredCache(RsCacheData &data); /* use pid/cid in data */
/*!
*
* @param data all cache store by cachestore is store here
* @return false not returned, only true at the moment
*/
bool getAllStoredCaches(std::list<RsCacheData> &data); /* use pid/cid in data */
/*!
* input from CacheStrapper -> store can then download new data
*/
void availableCache(const RsCacheData &data);
/*!
* should be called when the download is completed ... cache data is loaded
*/
void downloadedCache(const RsCacheData &data);
/*!
* called if the download fails, TODO: nothing done yet
*/
void failedCache(const RsCacheData &data);
/* virtual functions overloaded by cache implementor */
/* controls if peer is an accepted provider for cache items. Default is yes. To be overloaded. */
virtual bool isPeerAcceptedAsCacheProvider(const RsPeerId& /*peer_id*/) { return true ; }
/*!
* @param data cache data is stored here
* @return false is failed (cache does not exist), otherwise true
*/
virtual bool fetchCache(const RsCacheData &data); /* a question? */
virtual int nameCache(RsCacheData &data); /* fill in the name/path */
virtual int loadCache(const RsCacheData &data); /* actual load, once data available */
/* get private data */
std::string getCacheDir() { return cacheDir; }
bool isMultiCache() { return multiCache; }
uint16_t getCacheType() { return cacheType; }
/*!
* display, e.g. can pass std::out, cerr, ofstream, etc
*/
void listCaches(std::ostream &out);
protected:
/*!
* ** MUTEX LOCKING
*/
void lockData() const;
/*!
* ** MUTEX LOCKING
*/
void unlockData() const;
/*! This function is called to store Cache Entry in the CacheStore Table.
* it must be called from within a Mutex Lock....
*
* It doesn't lock itself -> to avoid race conditions
*/
void locked_storeCacheEntry(const RsCacheData &data);
/*! This function is called to store Cache Entry in the CacheStore Table.
* it must be called from within a Mutex Lock....
*
* It doesn't lock itself -> to avoid race conditions
*/
bool locked_getStoredCache(RsCacheData &data);
private:
uint16_t cacheType; /* for checking */
bool multiCache; /* do we care about subid's */
CacheStrapper *mStrapper;
CacheTransfer *cacheTransfer;
std::string cacheDir;
mutable RsMutex cMutex;
std::map<RsPeerId, CacheSet> caches;
};
/***************************** CacheStrapper *****************************/
/*!
* a convenient to pass cache handles to cachestrapper to maintain the cache service
* Make Sure you get the Ids right! see rsservicesids.h.
* When creating a cache service this data structure is
* source, usually the child class of store and source also serves as both handles
* @see CacheStrapper
*/
class CachePair
{
public:
/*!
* Default constructor, all variables set to NULL
*/
CachePair()
:source(NULL), store(NULL), id(0, 0) { return; }
/*!
*
* @param a cache source for service
* @param b cache store for service
* @param c the cache service id, c.type should be set to service id service-child class of store and source
*/
CachePair(CacheSource *a, CacheStore *b, CacheId c)
:source(a), store(b), id(c) { return; }
CacheSource *source;
CacheStore *store;
CacheId id; /// should be set id type to service types of store and source service-child class, and subid for multicache use
};
bool operator<(const CachePair &a, const CachePair &b);
/*!
* CacheStrapper: maintains a set of CacheSources, and CacheStores,
* queries and updates as new information arrives.
*/
class p3ServiceControl;
class CacheStrapper: public pqiServiceMonitor, public p3Config
{
public:
/*!
* @param cm handle used by strapper for getting peer connection information (online peers, sslids...)
* @return
*/
CacheStrapper(p3ServiceControl *sc, uint32_t ftServiceId);
virtual ~CacheStrapper() { return; }
/************* from pqiMonitor *******************/
virtual void statusChange(const std::list<pqiServicePeer> &plist);
/************* from pqiMonitor *******************/
/* Feedback from CacheSources */
/*!
* send data to peers online and self
* @param data
*
*/
void refreshCache(const RsCacheData &data);
void refreshCache(const RsCacheData &data,const std::set<RsPeerId>& destination_peers); // specify a particular list of destination peers (self not added!)
/*!
* forces config savelist
* @param data
* @see saveList()
*/
void refreshCacheStore(const RsCacheData &data);
/*!
* list of Caches to send out
*/
bool getCacheUpdates(std::list<std::pair<RsPeerId, RsCacheData> > &updates);
/*!
* add to strapper's cachepair set so a related service's store and source can be maintained
* @param pair the source and store handle for a service
*/
void addCachePair(CachePair pair);
/*** I/O (2) ***/
void recvCacheResponse(RsCacheData &data, time_t ts);
void handleCacheQuery(const RsPeerId& id, std::map<CacheId, RsCacheData> &data);
/*!
* search through CacheSources.
* @return false if cachedate mapping to hash not found
*/
bool findCache(const RsFileHash &hash, RsCacheData &data) const;
/* display */
void listCaches(std::ostream &out);
/*!
* does not do anything
* @param out
* @deprecated
*/
void listPeerStatus(std::ostream &out);
/**
* Checks if the cache physically exist at path given
* @param data
* @return whether it exists or not
*/
bool CacheExist(RsCacheData& data);
/* Config */
protected:
/* Key Functions to be overloaded for Full Configuration */
virtual RsSerialiser *setupSerialiser();
virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
virtual bool loadList(std::list<RsItem *>& load);
private:
/* these are static - so shouldn't need mutex */
p3ServiceControl *mServiceCtrl;
uint32_t mFtServiceId;
std::map<uint16_t, CachePair> caches;
RsMutex csMtx; /* protect below */
std::list<std::pair<RsPeerId, RsCacheData> > mCacheUpdates;
};
#endif

63
libretroshare/src/dbase/cachetest.h
View File

@ -1,63 +0,0 @@
/*
* RetroShare FileCache Module: cachetest.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_TEST_CACHE_H
#define MRK_TEST_CACHE_H
#include "cachestrapper.h"
#define TESTID 0xffff
#define TESTID2 0xffee
class CacheTestSource: public CacheSource
{
public:
CacheTestSource(CacheStrapper *cs, std::string dir)
:CacheSource(cs, TESTID, false, dir) { return; }
};
class CacheTestStore: public CacheStore
{
public:
CacheTestStore(CacheTransfer *cft, std::string dir)
:CacheStore(TESTID, false, cft, dir) { return; }
};
class CacheTestMultiSource: public CacheSource
{
public:
CacheTestMultiSource(CacheStrapper *cs, std::string dir)
:CacheSource(cs, TESTID2, true, dir) { return; }
};
class CacheTestMultiStore: public CacheStore
{
public:
CacheTestMultiStore(CacheTransfer *cft, std::string dir)
:CacheStore(TESTID2, true, cft, dir) { return; }
};
#endif

1706
libretroshare/src/dbase/fimonitor.cc
View File

File diff suppressed because it is too large Load Diff

220
libretroshare/src/dbase/fimonitor.h
View File

@ -1,220 +0,0 @@
/*
* RetroShare FileCache Module: fimonitor.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef FILE_INDEX_MONITOR_H
#define FILE_INDEX_MONITOR_H
#include "dbase/cachestrapper.h"
#include "dbase/findex.h"
#include "util/rsthreads.h"
#include "retroshare/rsfiles.h"
/******************************************************************************************
* The Local Monitoring Class: FileIndexMonitor.
*
* This periodically scans the directory tree, and updates any modified directories/files.
*
*****************************************************************************************/
/******************************************************************************************
STILL TODO:
(1) Implement Hash function.
bool FileIndexMonitor::hashFile(std::string path, FileEntry &fi);
(2) Add Shared directory controls to Monitor.
int FileIndexMonitor::addSharedDirectory(std::path);
int FileIndexMonitor::removeSharedDirectory(std::path);
std::string FileIndexMonitor::findRealRoot(std::string base);
These must be split into <base>/<top> and the mapping saved.
eg: addSharedDirectory("c:/home/stuff/dir1") --> "c:/home/stuff" <-> "dir1"
This code has been written already, and can just be moved over.
FOR LATER:
(2) Port File/Directory lookup code to windoze. (or compile dirent.c under windoze)
(3) Add Load/Store interface to FileIndexMonitor. (later)
(4) Integrate with real Thread/Mutex code (last thing to do)
******************************************************************************************/
class DirContentToHash
{
public:
std::vector<FileEntry> fentries ;
std::string realpath ;
std::string dirpath ;
};
class HashCache
{
public:
HashCache(const std::string& save_file_name) ;
void save() ;
void insert(const std::string& full_path,uint64_t size,time_t time_stamp,const RsFileHash& hash) ;
bool find(const std::string& full_path,uint64_t size,time_t time_stamp,RsFileHash& hash) ;
void clean() ;
typedef struct
{
uint64_t size ;
uint64_t time_stamp ;
uint64_t modf_stamp ;
RsFileHash hash ;
} HashCacheInfo ;
void setRememberHashFilesDuration(uint32_t days) { _max_cache_duration_days = days ; }
uint32_t rememberHashFilesDuration() const { return _max_cache_duration_days ; }
void clear() { _files.clear(); }
bool empty() const { return _files.empty() ; }
private:
uint32_t _max_cache_duration_days ; // maximum duration of un-requested cache entries
std::map<std::string, HashCacheInfo> _files ;
std::string _path ;
bool _changed ;
};
/******************************************************************************************
* FileIndexMonitor
*****************************************************************************************/
class FileIndexMonitor: public CacheSource, public RsTickingThread
{
public:
FileIndexMonitor(CacheStrapper *cs, std::string cachedir, const RsPeerId& pid, const std::string& config_dir);
virtual ~FileIndexMonitor();
/* external interface for filetransfer */
bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list<RsNodeGroupId>& parent_groups) const;
int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) ;
int SearchBoolExp(Expression *exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
int filterResults(std::list<FileEntry*>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
/* external interface for local access to files */
bool convertSharedFilePath(std::string path, std::string &fullpath);
/* Interacting with CacheSource */
/* overloaded from CacheSource */
virtual bool loadLocalCache(const RsCacheData &data); /* called with stored data */
bool updateCache(const RsCacheData &data,const std::set<RsPeerId>& dest_peers); /* we call when we have a new cache for others */
/* the FileIndexMonitor inner workings */
//virtual void run(std::string& currentJob); /* overloaded from RsThread */
//void updateCycle(std::string& currentJob);
virtual void data_tick(); /* overloaded from RsThread */
void updateCycle();
// Interface for browsing dir hirarchy
int RequestDirDetails(void*, DirDetails&, FileSearchFlags) const ;
uint32_t getType(void*) const ;
int RequestDirDetails(const std::string& path, DirDetails &details) const ;
// set/update shared directories
virtual void setSharedDirectories(const std::list<SharedDirInfo>& dirs);
void getSharedDirectories(std::list<SharedDirInfo>& dirs);
void updateShareFlags(const SharedDirInfo& info) ;
void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed
void forceDirListsRebuildAndSend() ; // Force re-build dir lists because groups have changed. Does not re-check files.
bool inDirectoryCheck();
/* util fns */
// from CacheSource
virtual bool cachesAvailable(const RsPeerId& pid, std::map<CacheId, RsCacheData> &ids) ;
protected:
// Sets/gets the duration period within which already hashed files are remembered.
//
void setRememberHashFilesDuration(uint32_t days) ;
uint32_t rememberHashFilesDuration() const ;
void setRememberHashFiles(bool) ;
bool rememberHashFiles() ;
// Remove any memory of formerly hashed files that are not shared anymore
void clearHashFiles() ;
void setPeriod(int insecs);
int getPeriod() const;
bool autoCheckEnabled() const ;
private:
/* the mutex should be locked before calling these 3. */
// Saves file indexs and update the cache. Returns the name of the main
// file index, which becomes the new reference file for mod times.
//
time_t locked_saveFileIndexes(bool update_cache) ;
// Finds the share flags associated with this file entry.
void locked_findShareFlagsAndParentGroups(FileEntry *fe, FileStorageFlags& shareflags, std::list<RsNodeGroupId> &parent_groups) const ;
std::string locked_findRealRoot(std::string base) const;
void hashFiles(const std::vector<DirContentToHash>& to_hash) ;
bool hashFile(std::string path, FileEntry &fi); /* To Implement */
/* data */
mutable RsMutex fiMutex;
FileIndex fi;
int updatePeriod;
std::map<std::string, SharedDirInfo> directoryMap; /* used by findRealRoot */
/* flags to kick - if we were busy or sleeping */
bool pendingDirs;
bool pendingForceCacheWrite;
/* flags to force Check, to tell if we're in check */
bool mForceCheck;
bool mInCheck;
std::list<SharedDirInfo> pendingDirList;
bool internal_setSharedDirectories();
HashCache hashCache ;
bool useHashCache ;
std::map<RsPeerId,RsCacheData> _cache_items_per_peer ; // stored the cache items to be sent to each peer.
// This file is the location of the current index file. When checking for new files, we compare the modification time
// of this file to the mod time of the files on the disk. This allows to now account for time-shift in the computer.
//
time_t reference_time ;
};
#endif

1516
libretroshare/src/dbase/findex.cc
View File

File diff suppressed because it is too large Load Diff

278
libretroshare/src/dbase/findex.h
View File

@ -1,278 +0,0 @@
/*
* RetroShare FileCache Module: findex.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef FILE_INDEX_H
#define FILE_INDEX_H
#include <string>
#include <map>
#include <set>
#if __MACH__
#include <unordered_set>
#else
#include <tr1/unordered_set>
#endif
#include <list>
#include <vector>
#include <stdint.h>
#include "retroshare/rstypes.h"
/******************************************************************************************
* The Key Data Types for the File Index:
FileEntry : Information about a single file.
DirEntry : Information about a directory and its children
PersonEntry : Information the root of a FileIndex.
FileIndex : A Collection of root directories, with a set of functions to manipulate them.
In terms of retroshare, There will be a single 'Local' FileIndex used to store
the shared files, and a set of 'Remote' FileIndices which are used to store
the available files of all the peers.
******************************************************************************************/
/******************************************************************************************
STILL TODO:
(1) Load/Store a FileIndex to file...
int FileIndex::loadIndex(FILE *input);
int FileIndex::saveIndex(FILE *input);
This can be done in a recursive manner, or handled completely within FileIndex.
(2) Search Functions for Partial File Names and Hashes.
int FileIndex::searchHash(std::string hash, std::list<FileEntry> &results);
int FileIndex::searchTerms(std::list<string> terms, std::list<FileEntry> &results);
This is probably best done in a recursive manner.
The search could also be extended to handle complex Boolean searches such as :
match (size > 100K) && (name contains 'Blue') .... if anyone is interested.
But this can get quite complicated, and can be left to a later date.
******************************************************************************************/
/******************************************************************************************
* FileEntry
*****************************************************************************************/
#include <util/smallobject.h>
class DirEntry;
class FileEntry: public RsMemoryManagement::SmallObject
{
public:
FileEntry()
: size(0), modtime(0), pop(0), updtime(0), parent(NULL), row(0)
{ return; }
virtual ~FileEntry() { return; }
virtual uint32_t type() const { return DIR_TYPE_FILE ; }
virtual int print(std::string &out);
/* Data */
std::string name;
RsFileHash hash;
uint64_t size; /* file size */
time_t modtime; /* modification time - most recent mod time for a sub entry for dirs */
int pop; /* popularity rating */
time_t updtime; /* last updated */
/* References for easy manipulation */
DirEntry *parent;
int row;
std::list<std::string> parent_groups ;
};
/******************************************************************************************
* DirEntry
*****************************************************************************************/
class DirEntry: public FileEntry
{
public:
DirEntry() : most_recent_time(0) {}
/* cleanup */
virtual ~DirEntry();
/* update local entries */
DirEntry * updateDir(const FileEntry& fe, time_t updtime);
FileEntry * updateFile(const FileEntry& fe, time_t updtime);
virtual uint32_t type() const { return DIR_TYPE_DIR ; }
int checkParentPointers();
int updateChildRows();
/* remove local entries */
int removeFile(const std::string& name);
int removeDir(const std::string& name);
int removeOldDir(const std::string& name, time_t old); /* checks ts first */
/* recursive cleanup */
int removeOldEntries(time_t old, bool recursive);
/* recursive searches */
DirEntry * findOldDirectory(time_t old);
DirEntry * findDirectory(const std::string& path);
/* recursive update directory mod/pop values */
int updateDirectories(const std::string& path, int pop, int modtime);
/* output */
int print(std::string &out);
int saveEntry(std::string &out);
void writeDirInfo(std::string&);
void writeFileInfo(std::string&);
/* Data */
std::string path; /* full path (includes name) */
std::map<std::string, DirEntry *> subdirs;
std::map<std::string, FileEntry *> files;
time_t most_recent_time; /* last updated */
/* Inherited members from FileEntry:
int size - count for dirs
std::string name; - directory name
std::string hash; - not used
int size; - not used
int modtime; - most recent modication time of any child file (recursive)
int pop; - most popular child file (recursive)
int updtime; - last updated
*/
};
/******************************************************************************************
* PersonEntry
*****************************************************************************************/
class PersonEntry: public DirEntry
{
public:
/* cleanup */
PersonEntry(const RsPeerId& pid) : id(pid) { return; }
virtual ~PersonEntry() { return; }
DirEntry &operator=(DirEntry &src)
{
DirEntry *pdest = this;
(*pdest) = src;
return *this;
}
virtual uint32_t type() const { return DIR_TYPE_PERSON ; }
/* Data */
RsPeerId id;
/* Inherited members from FileEntry:
int size - count for dirs
std::string name; - directory name
std::string hash; - not used
int size; - not used
int modtime; - most recent modication time of any child file (recursive)
int pop; - most popular child file (recursive)
int updtime; - last updated
*/
};
/******************************************************************************************
* FileIndex
*****************************************************************************************/
class Expression;
class FileIndex
{
public:
FileIndex(const RsPeerId& pid);
~FileIndex();
/* control root entries */
int setRootDirectories(const std::list<std::string> &inlist, time_t utime);
int getRootDirectories(std::list<std::string> &outlist);
/* update (index building) */
DirEntry * updateDirEntry(const std::string& path, const FileEntry& fe, time_t utime);
FileEntry * updateFileEntry(const std::string& path, const FileEntry& fe, time_t utime);
DirEntry * findOldDirectory(time_t old); /* finds directories older than old */
int removeOldDirectory(const std::string& fpath, const std::string& name, time_t old);
int cleanOldEntries(time_t old); /* removes entries older than old */
/* debug */
int printFileIndex(std::string &out);
int printFileIndex(std::ostream &out);
/* load/save to file */
int loadIndex(const std::string& filename, const RsFileHash &expectedHash, uint64_t size);
int saveIndex(const std::string& filename, RsFileHash &fileHash, uint64_t &size, const std::set<std::string>& forbidden_roots);
/* search through this index */
int searchTerms(const std::list<std::string>& terms, std::list<FileEntry *> &results) const;
int searchHash(const RsFileHash& hash, std::list<FileEntry *> &results) const;
int searchBoolExp(Expression * exp, std::list<FileEntry *> &results) const;
/// Recursively compute the maximum modification time of children.
/// Used to easily retrieve mose recent files.
//
void updateMaxModTime() ;
void RecursUpdateMaxModTime(DirEntry *) ;
PersonEntry *root;
#ifdef __MACH__
static std::unordered_set<void*> _pointers ;
#else
static std::tr1::unordered_set<void*> _pointers ;
#endif
static void registerEntry(void*p) ;
static void unregisterEntry(void*p) ;
static bool isValid(void*p) ;
/// Fills up details from the data contained in ref.
//
static bool extractData(void *ref,DirDetails& details) ;
static uint32_t getType(void *ref) ;
void *findRef(const std::string& path) const ;
bool extractData(const std::string& path,DirDetails& details) const ;
void updateHashIndex() ;
void recursUpdateHashIndex(DirEntry *) ;
std::map<RsFileHash,FileEntry*> _file_hashes ;
};
#endif

459
libretroshare/src/dbase/fistore.cc
View File

@ -1,459 +0,0 @@
/*
* RetroShare FileCache Module: fistore.cc
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include <time.h>
#include "rsserver/p3face.h"
#include "dbase/fistore.h"
#include "retroshare/rsexpr.h"
#include "retroshare/rsfiles.h"
#include "serialiser/rsserviceids.h"
#include "pqi/p3peermgr.h"
#include "pqi/p3notify.h"
FileIndexStore::FileIndexStore(CacheStrapper *cs, CacheTransfer *cft,
p3PeerMgr *cnmgr, RsPeerId ownid, std::string cachedir)
:CacheStore(RS_SERVICE_TYPE_FILE_INDEX, false, cs, cft, cachedir),
localId(ownid), localindex(NULL), mPeerMgr(cnmgr)
{
return;
}
FileIndexStore::~FileIndexStore()
{
/* clean up the Index */
return;
}
/***
* #define FIS_DEBUG2 1
* #define FIS_DEBUG 1
**/
/* actual load, once data available */
int FileIndexStore::loadCache(const RsCacheData &data)
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() hash: " << data.hash << std::endl;
std::cerr << "FileIndexStore::loadCache() path: " << data.path << std::endl;
std::cerr << "FileIndexStore::loadCache() name: " << data.name << std::endl;
std::cerr << "FileIndexStore::loadCache() size: " << data.size << std::endl;
#endif
/* do Callback */
AboutToModify();
/* lock it up */
lockData();
FileIndex *fiold = NULL;
bool local = (data.pid == localId);
std::map<RsPeerId, FileIndex *>::iterator it;
/* remove old cache */
if (local)
{
fiold = localindex;
localindex = NULL;
}
else if (indices.end() != (it = indices.find(data.pid)))
{
fiold = it->second;
indices.erase(it);
//delete fi;
}
if(mPeerMgr->isFriend(data.pid))
{
// We discard file lists from non friends. This is the place to remove file lists of deleted friends
// from the cache. Doing this, the file list still shows in a session where we deleted a friend, but will be removed
// at next restart.
//
/* load Cache */
FileIndex *finew = new FileIndex(data.pid);
if (finew->loadIndex(data.path + '/' + data.name, data.hash, data.size))
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() Succeeded!" << std::endl;
#endif
/* This is not the place to set the peername.
* It is a hack, which makes it programmatically impossible
* to get the file data out....
*
* peername should not be used in dbase.
*/
finew->root->name = data.pid.toStdString();
if (local)
{
localindex = finew;
}
else
{
indices[data.pid] = finew;
}
delete fiold;
/* store in tale */
locked_storeCacheEntry(data);
}
else
{
#ifdef FIS_DEBUG2
std::cerr << "FileIndexStore::loadCache() Failed!" << std::endl;
#endif
/* reinstall the old one! */
delete finew;
if (fiold)
{
if (local)
{
localindex = fiold;
}
else
{
indices[data.pid] = fiold;
}
}
}
}
#ifdef FIS_DEBUG
else
std::cerr << "Discarding file list from deleted peer " << data.pid << std::endl ;
#endif
/* need to correct indices(row) for the roots of the FileIndex */
int i = 0;
for(it = indices.begin(); it != indices.end(); ++it)
{
(it->second)->root->row = i++;
it->second->FileIndex::updateMaxModTime() ;
}
if (localindex)
{
localindex->root->row = 0;
localindex->updateMaxModTime() ;
}
unlockData();
ModCompleted();
bool ret = false;
return ret;
}
/* Search Interface - For Directory Access */
int FileIndexStore::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails& details) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator it = indices.find(uid);
bool found = true;
if (it != indices.end())
found = it->second->extractData(path,details) ;
unlockData();
return found ;
#ifdef OLD_STUFF_TO_REMOVE
/* lock it up */
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator it;
it = indices.find(uid);
bool found = true;
if (it != indices.end())
{
//DirEntry *fdir = (it->second).lookupDirectory(path);
/* translate it
*/
bool b = FileIndex::extractData((it->second)->root,details) ;
found = found && b ;
}
else
found = false;
unlockData();
return found;
#endif
}
int FileIndexStore::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const
{
/* remove unused parameter warnings */
(void) flags;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() ref=" << ref << " flags: " << flags << std::endl;
#endif
std::map<RsPeerId, FileIndex *>::const_iterator pit;
lockData();
// checked by FileIndex::extractData
// if(ref != NULL && !FileIndex::isValid(ref))
// {
// unlockData() ;
// return false ;
// }
/* so cast *ref to a DirEntry */
/* root case */
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() CHKS" << std::endl;
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
(pit->second)->root->checkParentPointers();
}
#endif
if (ref == NULL)
{
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::RequestDirDetails() ref=NULL (root)" << std::endl;
#endif
/* get remote root entries */
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
/*
*/
FileIndex *fileIndex = pit->second;
DirStub stub;
stub.type = DIR_TYPE_PERSON;
stub.name = fileIndex->root->name;
stub.ref = fileIndex->root;
details.children.push_back(stub);
}
details.parent = NULL;
details.prow = -1;
details.ref = NULL;
details.type = DIR_TYPE_ROOT;
details.name = "";
details.hash.clear() ;
details.path = "";
details.count = indices.size();
details.age = 0;
details.flags.clear() ;
details.min_age = 0;
unlockData();
return true ;
}
bool b = FileIndex::extractData(ref,details) ;
unlockData();
return b;
}
uint32_t FileIndexStore::getType(void *ref) const
{
lockData() ;
uint32_t b = FileIndex::getType(ref) ;
unlockData();
return b;
}
int FileIndexStore::SearchHash(const RsFileHash& hash, std::list<FileDetail> &results) const
{
lockData();
results.clear() ;
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
time_t now = time(NULL);
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash()" << std::endl;
#endif
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash() Searching: Peer ";
std::cerr << pit->first << std::endl;
#endif
firesults.clear();
(pit->second)->searchHash(hash, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
FileDetail fd;
fd.id = pit->first;
fd.name = (*rit)->name;
fd.hash = (*rit)->hash;
fd.path = ""; /* TODO */
fd.size = (*rit)->size;
fd.age = now - (*rit)->modtime;
fd.rank = (*rit)->pop;
results.push_back(fd);
}
}
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchHash() Found " << results.size();
std::cerr << " Results from " << indices.size() << " Peers" << std::endl;
#endif
unlockData();
return results.size();
}
int FileIndexStore::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
results.clear() ;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::SearchKeywords()" << std::endl;
#endif
if(flags & RS_FILE_HINTS_REMOTE)
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
firesults.clear();
(pit->second)->searchTerms(keywords, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!FileIndex::extractData(*rit, dd))
continue ;
results.push_back(dd);
}
}
if(flags & RS_FILE_HINTS_LOCAL)
if (localindex)
{
firesults.clear();
localindex->searchTerms(keywords, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
if(!FileIndex::extractData(*rit, dd))
continue ;
dd.id.clear() ;
results.push_back(dd);
}
}
unlockData();
return results.size();
}
int FileIndexStore::searchBoolExp(Expression * exp, std::list<DirDetails> &results) const
{
lockData();
std::map<RsPeerId, FileIndex *>::const_iterator pit;
std::list<FileEntry *>::iterator rit;
std::list<FileEntry *> firesults;
#ifdef FIS_DEBUG
std::cerr << "FileIndexStore::searchBoolExp()" << std::endl;
#endif
for(pit = indices.begin(); pit != indices.end(); ++pit)
{
firesults.clear();
(pit->second)->searchBoolExp(exp, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
FileIndex::extractData(*rit, dd);
results.push_back(dd);
}
}
/* finally search local files */
if (localindex)
{
firesults.clear();
localindex->searchBoolExp(exp, firesults);
/* translate results */
for(rit = firesults.begin(); rit != firesults.end(); ++rit)
{
DirDetails dd;
FileIndex::extractData(*rit, dd);
dd.id.clear() ;
results.push_back(dd);
}
}
unlockData();
return results.size();
}
int FileIndexStore::AboutToModify()
{
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
return 1;
}
int FileIndexStore::ModCompleted()
{
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_FRIENDS, 0);
return 1;
}

102
libretroshare/src/dbase/fistore.h
View File

@ -1,102 +0,0 @@
/*
* RetroShare FileCache Module: fistore.h
*
* Copyright 2004-2007 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_FILE_INDEX_STORE_H
#define MRK_FILE_INDEX_STORE_H
/**********
* Stores the FileCaches of the Peers
* must implement 'loadCache' to
*
* This class is also accessed by the GUI....
* and the FileTransfer class.
*
*/
class p3PeerMgr ;
#include "dbase/findex.h"
#include "dbase/cachestrapper.h"
#include "retroshare/rsiface.h"
class FileStoreResult
{
public:
std::string id;
std::string path;
std::string hash;
std::string name;
};
class NotifyCallback
{
public:
NotifyCallback() { return; }
virtual ~NotifyCallback() { return; }
virtual void AboutToModify() { return; }
virtual void ModCompleted() { return; }
};
class Expression;
class FileIndexStore: public CacheStore
{
public:
FileIndexStore(CacheStrapper *cs, CacheTransfer *cft, p3PeerMgr *pmgr, RsPeerId ownid, std::string cachedir);
virtual ~FileIndexStore();
/* virtual functions overloaded by cache implementor */
virtual int loadCache(const RsCacheData &data); /* actual load, once data available */
/* Search Interface - For FileTransfer Lookup */
int SearchHash(const RsFileHash &hash, std::list<FileDetail> &results) const;
/* Search Interface - For Search Interface */
int SearchKeywords(std::list<std::string> terms, std::list<DirDetails> &results,FileSearchFlags flags) const;
/* Search Interface - for Adv Search Interface */
int searchBoolExp(Expression * exp, std::list<DirDetails> &results) const;
/* Search Interface - For Directory Access */
int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) const;
int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) const;
uint32_t getType(void *ref) const ;
private:
int AboutToModify();
int ModCompleted();
std::map<RsPeerId, FileIndex *> indices;
RsPeerId localId;
FileIndex *localindex;
p3PeerMgr *mPeerMgr ;
};
#endif

289
libretroshare/src/dbase/rsexpr.cc
View File

@ -1,289 +0,0 @@
/*
* rs-core/src/dbase: rsexpr.cc
*
* RetroShare C++ Interface.
*
* Copyright 2007-2008 by Kashif Kaleem.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "dbase/findex.h"
#include "retroshare/rsexpr.h"
#include <algorithm>
#include <functional>
/******************************************************************************************
eval functions of relational expressions.
******************************************************************************************/
template<>
void RelExpression<int>::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(LowerValue) ;
e._ints.push_back(HigherValue) ;
}
bool DateExpression::eval(FileEntry *file)
{
return evalRel(file->modtime);
}
bool SizeExpressionMB::eval(FileEntry *file)
{
return evalRel((int)(file->size/(uint64_t)(1024*1024)));
}
bool SizeExpression::eval(FileEntry *file)
{
return evalRel(file->size);
}
bool PopExpression::eval(FileEntry *file)
{
return evalRel(file->pop);
}
/******************************************************************************************
Code for evaluating string expressions
******************************************************************************************/
bool NameExpression::eval(FileEntry *file)
{
return evalStr(file->name);
}
bool PathExpression::eval(FileEntry *file){
std::string path;
/*Construct the path of this file*/
DirEntry * curr = file->parent;
while ( curr != NULL ){
path = curr->name+"/"+ path;
curr = curr->parent;
}
return evalStr(path);
}
bool ExtExpression::eval(FileEntry *file){
std::string ext;
/*Get the part of the string after the last instance of . in the filename */
size_t index = file->name.find_last_of('.');
if (index != std::string::npos) {
ext = file->name.substr(index+1);
if (ext != "" ){
return evalStr(ext);
}
}
return false;
}
bool HashExpression::eval(FileEntry *file){
return evalStr(file->hash.toStdString());
}
/*Check whether two strings are 'equal' to each other*/
static bool StrEquals(const std::string & str1, const std::string & str2,
bool IgnoreCase ){
if ( str1.size() != str2.size() ){
return false;
} else if (IgnoreCase) {
std::equal( str1.begin(), str1.end(),
str2.begin(), CompareCharIC() );
}
return std::equal( str1.begin(), str1.end(),
str2.begin());
}
/*Check whether one string contains the other*/
static bool StrContains( const std::string & str1, const std::string & str2,
bool IgnoreCase){
std::string::const_iterator iter ;
if (IgnoreCase) {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end(), CompareCharIC() );
} else {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end());
}
return ( iter != str1.end() );
}
bool StringExpression :: evalStr ( const std::string &str ){
std::list<std::string>::iterator iter;
switch (Op) {
case ContainsAllStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str, *iter, IgnoreCase) == false ){
return false;
}
}
return true;
break;
case ContainsAnyStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
case EqualsString:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrEquals (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
default:
return false;
}
return false;
}
/*************************************************************************
* linearization code
*************************************************************************/
void CompoundExpression::linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_COMP) ;
e._ints.push_back(Op) ;
Lexp->linearize(e) ;
Rexp->linearize(e) ;
}
void StringExpression::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(IgnoreCase) ;
e._ints.push_back(terms.size()) ;
for(std::list<std::string>::const_iterator it(terms.begin());it!=terms.end();++it)
e._strings.push_back(*it) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e)
{
int i=0,j=0,k=0 ;
return toExpr(e,i,j,k) ;
}
void LinearizedExpression::readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list<std::string>& strings,bool& b,StringOperator& op)
{
op = static_cast<StringOperator>(e._ints[n_ints++]) ;
b = e._ints[n_ints++] ;
int n = e._ints[n_ints++] ;
strings.clear() ;
for(int i=0;i<n;++i)
strings.push_back(e._strings[n_strings++]) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e,int& n_tok,int& n_ints,int& n_strings)
{
LinearizedExpression::token tok = static_cast<LinearizedExpression::token>(e._tokens[n_tok++]) ;
switch(tok)
{
case EXPR_DATE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new DateExpression(op,lv,hv) ;
}
case EXPR_POP: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new PopExpression(op,lv,hv) ;
}
case EXPR_SIZE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpression(op,lv,hv) ;
}
case EXPR_HASH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new HashExpression(op,strings) ;
}
case EXPR_NAME: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new NameExpression(op,strings,b) ;
}
case EXPR_PATH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_EXT: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_COMP: {
LogicalOperator op = static_cast<LogicalOperator>(e._ints[n_ints++]) ;
Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ;
Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ;
return new CompoundExpression(op,e1,e2) ;
}
case EXPR_SIZE_MB: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpressionMB(op,lv,hv) ;
}
default:
std::cerr << "No expression match the current value " << tok << std::endl ;
return NULL ;
}
}

130
libretroshare/src/file_sharing/README.txt
View File

@ -40,9 +40,35 @@ Big picture
- the same file should be able to be held by two different directories. Hash search will return a single hit.
- the previously existing PersonEntry had no field and was overloading DirEntry, with overwritten file names, hashes etc. Super bad!
Directory storage file format
-----------------------------
* should be extensible (xml or binary format? Binary, because it's going to be encrypted anyway)
=> works with binary fields
=>
[= version =]
[= peer id =]
[= num entries =]
[= some information =]
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
...
2 1-5 v 2 1-5 v
* entry content
Tag | Content | Size
----------------+--------------------------------------+------
01 | sha1 hash | 20
01 | sha1^2 hash | 20
02 | file name | < 512
03 | file size | 8
04 | dir name | < 512
05 | last modif time local | 4
06 | last modif time including sub-dirs | 4
Classes
-------
Rs
p3ShareManager
- tick()
@ -75,6 +101,29 @@ Classes
- parent groups
- group flags
Best data structure for file index
----------------------------------
| Hash map map list
----------------+-----------------+------------+--------------
Adding | Constant | log(n) | O(n)
Hash search | Constant | log(n) | O(n)
Name/exp search | O(n) | O(n) | O(n)
Recursive browse| Constant | log(n) | O(n)
Should we use the same struct for files and directories?
Sol 1:
DirClass + PersonClass + FileEntry class
- each has pointers to elements list of the same type
- lists are handled for Files (all file entries),
Directories are represented by the hash of the full path
Sol 2:
Same class for all elements, in a single hash map. Each element is
defined by its type (Dir, Person, File) which all have a hash.
Syncing between peers
---------------------
@ -83,19 +132,84 @@ Generating sync events
- for each directory, in breadth first order
- if directory has changed, or last update is old
=> push a sync request
- store the peer's last up time. Compare with peer uptimes recursively.
* Server side
- after a change, broadcast a "directory changed" packet to all connected friends
* directoy updater
- crawl through directories
- compare TS of files, missing files, new files
- feed a queue of files to hash
- directory whatcher gets notified when files are hashed
- a separate component hashes files (FileHashingProcess)
DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy)
| |
| |
| |
+-----------------------+------------------+
| |
Shared File Service |
| |
| |
+----------- own file list -------+---------- Encrypted/compressed save to disk
| | |
+----------- friend file lists ---+
Roadmap
-------
- complete this file until a proper description of the whole thing is achieved.
- create a new directory and implement the .h for the basic functionality
- look into existing code in ftServer for the integration, but don't change anything yet
- setup class hierarchy
- merge hash cache into file lists.
[X] complete this file until a proper description of the whole thing is achieved.
[X] create a new directory and implement the .h for the basic functionality
[ ] look into existing code in ftServer for the integration, but don't change anything yet
[X] setup class hierarchy
[ ] merge hash cache into file lists.
[ ] new format for saving of FileIndex to make it locally encrypted, compact and extensible
[ ] create basic directory functionality with own files: re-hash, and store
[ ] display own files in GUI, with proper update and working sort
TODO
====
[ ] directory handler
[ ] abstract functions to keep a directory and get updates to it.
[ ] hierarchical storage representation.
[ ] allow add/delete entries
[ ] auto-cleanup
[ ] directory updater
[ ] abstract layer
[ ] crawls the directory and ask updates
[ ] derive local directory updater
[ ] crawl local files, and asks updates to storage class
[ ] derive remote directory updater
[ ] crawl stored files, and request updates to storage class
[ ] load/save of directory content. Should be extensible
[ ] p3FileLists with minimal functonality: no exchange. Only storage of own file lists
[ ] service (items) for p3FileLists
[ ] connect RemoteDirModel to new system
[ ] test GUI functions
[ ] test update between peers
- optionally
- change the saving system of FileIndex to make it locally encrypted and compact

1099
libretroshare/src/file_sharing/dir_hierarchy.cc Normal file
View File

File diff suppressed because it is too large Load Diff

183
libretroshare/src/file_sharing/dir_hierarchy.h Normal file
View File

@ -0,0 +1,183 @@
/*
* RetroShare C++ Internal directory hierarchy class.
*
* file_sharing/dir_hierarchy.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <stdint.h>
#include <string.h>
#include <stdlib.h>
#include "directory_storage.h"
class InternalFileHierarchyStorage
{
public:
class FileStorageNode
{
public:
static const uint32_t TYPE_UNKNOWN = 0x0000 ;
static const uint32_t TYPE_FILE = 0x0001 ;
static const uint32_t TYPE_DIR = 0x0002 ;
virtual ~FileStorageNode() {}
virtual uint32_t type() const =0;
DirectoryStorage::EntryIndex parent_index;
uint32_t row ;
};
class FileEntry: public FileStorageNode
{
public:
FileEntry() : file_size(0), file_modtime(0) {}
FileEntry(const std::string& name,uint64_t size,time_t modtime) : file_name(name),file_size(size),file_modtime(modtime) {}
FileEntry(const std::string& name,uint64_t size,time_t modtime,const RsFileHash& hash) : file_name(name),file_size(size),file_modtime(modtime),file_hash(hash) {}
virtual uint32_t type() const { return FileStorageNode::TYPE_FILE ; }
virtual ~FileEntry() {}
// local stuff
std::string file_name ;
uint64_t file_size ;
time_t file_modtime;
RsFileHash file_hash ;
};
class DirEntry: public FileStorageNode
{
public:
DirEntry(const std::string& name) : dir_name(name), dir_modtime(0),dir_most_recent_time(0),dir_update_time(0) {}
virtual ~DirEntry() {}
virtual uint32_t type() const { return FileStorageNode::TYPE_DIR ; }
// local stuff
std::string dir_name ;
std::string dir_parent_path ;
RsFileHash dir_hash ;
std::vector<DirectoryStorage::EntryIndex> subdirs ;
std::vector<DirectoryStorage::EntryIndex> subfiles ;
time_t dir_modtime;
time_t dir_most_recent_time; // recursive most recent modification time, including files and subdirs in the entire hierarchy below.
time_t dir_update_time; // last time the information was updated for that directory. Includes subdirs indexes and subfile info.
};
// class stuff
InternalFileHierarchyStorage() ;
bool load(const std::string& fname) ;
bool save(const std::string& fname) ;
int parentRow(DirectoryStorage::EntryIndex e);
bool isIndexValid(DirectoryStorage::EntryIndex e) const;
bool getChildIndex(DirectoryStorage::EntryIndex e,int row,DirectoryStorage::EntryIndex& c) const;
bool updateSubDirectoryList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,time_t>& subdirs);
bool removeDirectory(DirectoryStorage::EntryIndex indx) ;
bool checkIndex(DirectoryStorage::EntryIndex indx,uint8_t type) const;
bool updateSubFilesList(const DirectoryStorage::EntryIndex& indx,const std::map<std::string,DirectoryStorage::FileTS>& subfiles,std::map<std::string,DirectoryStorage::FileTS>& new_files);
bool updateHash(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash);
bool updateFile(const DirectoryStorage::EntryIndex& file_index,const RsFileHash& hash, const std::string& fname,uint64_t size, const time_t modf_time);
bool updateDirEntry(const DirectoryStorage::EntryIndex& indx, const std::string& dir_name, time_t most_recent_time, time_t dir_modtime, const std::vector<RsFileHash> &subdirs_hash, const std::vector<FileEntry> &subfiles_array);
// TS get/set functions. Take one of the class members as argument.
bool getTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) const;
bool setTS(const DirectoryStorage::EntryIndex& index,time_t& TS,time_t DirEntry::* ) ;
// Do a complete recursive sweep over sub-directories and files, and update the lst modf TS. This could be also performed by a cleanup method.
time_t recursUpdateLastModfTime(const DirectoryStorage::EntryIndex& dir_index);
// hash stuff
bool getDirHashFromIndex(const DirectoryStorage::EntryIndex& index,RsFileHash& hash) const ;
bool getIndexFromDirHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ;
bool getIndexFromFileHash(const RsFileHash& hash,DirectoryStorage::EntryIndex& index) const ;
// file/dir access and modification
bool findSubDirectory(DirectoryStorage::EntryIndex e,const std::string& s) const ; // returns true when s is the name of a sub-directory in the given entry e
uint32_t mRoot ;
std::vector<FileStorageNode*> mNodes;// uses pointers to keep information about valid/invalid objects.
void compress() ; // use empty space in the vector, mostly due to deleted entries. This is a complicated operation, mostly due to
// all the indirections used. Nodes need to be moved, renamed, etc. The operation discards all file entries that
// are not referenced.
friend class DirectoryStorage ; // only class that can use this.
friend class LocalDirectoryStorage ; // only class that can use this.
// Low level stuff. Should normally not be used externally.
const FileStorageNode *getNode(DirectoryStorage::EntryIndex indx) const;
const DirEntry *getDirEntry(DirectoryStorage::EntryIndex indx) const;
const FileEntry *getFileEntry(DirectoryStorage::EntryIndex indx) const;
uint32_t getType(DirectoryStorage::EntryIndex indx) const;
DirectoryStorage::EntryIndex getSubFileIndex(DirectoryStorage::EntryIndex parent_index,uint32_t file_tab_index);
DirectoryStorage::EntryIndex getSubDirIndex(DirectoryStorage::EntryIndex parent_index,uint32_t dir_tab_index);
// search. SearchHash is logarithmic. The other two are linear.
bool searchHash(const RsFileHash& hash,std::list<DirectoryStorage::EntryIndex>& results);
int searchBoolExp(RsRegularExpression::Expression * exp, std::list<DirectoryStorage::EntryIndex> &results) const ;
int searchTerms(const std::list<std::string>& terms, std::list<DirectoryStorage::EntryIndex> &results) const ;
bool check(std::string& error_string) const ;// checks consistency of storage.
void print() const;
private:
void recursPrint(int depth,DirectoryStorage::EntryIndex node) const;
static bool nodeAccessError(const std::string& s);
static RsFileHash createDirHash(const std::string& dir_name,const std::string& dir_parent_path) ;
// Allocates a new entry in mNodes, possible re-using an empty slot and returns its index.
DirectoryStorage::EntryIndex allocateNewIndex();
// Removes the given subdirectory from the parent node and all its pendign subdirs. Files are kept, and will go during the cleaning
// phase. That allows to keep file information when moving them around.
bool recursRemoveDirectory(DirectoryStorage::EntryIndex dir);
// Map of the hash of all files. The file hashes are the sha1sum of the file data.
// is used for fast search access for FT.
// Note: We should try something faster than std::map. hash_map??
// Unlike directories, multiple files may have the same hash. So this cannot be used for anything else than FT.
std::map<RsFileHash,DirectoryStorage::EntryIndex> mFileHashes ;
// The directory hashes are the sha1sum of the
// full public path to the directory.
// The later is used by synchronisation items in order
// to avoid sending explicit EntryIndex values.
// This is kept separate from mFileHashes because the two are used
// in very different ways.
//
std::map<RsFileHash,DirectoryStorage::EntryIndex> mDirHashes ;
};

13
libretroshare/src/file_sharing/directory_list.h Normal file
View File

@ -0,0 +1,13 @@
// This class keeps a shared directory. It's quite the equivalent of the old "FileIndex" class
// The main difference is that it is
// - extensible
// - fast to search (at least for hashes). Should provide possibly multiple search handles for
// the same file, e.g. if connexion is encrypted.
// - abstracts the browsing in a same manner.
//
class SharedDirectoryList
{
public:
DirEntry mRoot ;
};

806
libretroshare/src/file_sharing/directory_storage.cc Normal file
View File

@ -0,0 +1,806 @@
/*
* RetroShare File list storage system.
*
* file_sharing/directory_storage.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include <set>
#include <time.h>
#include "serialiser/rstlvbinary.h"
#include "retroshare/rspeers.h"
#include "util/rsdir.h"
#include "util/rsstring.h"
#include "file_sharing_defaults.h"
#include "directory_storage.h"
#include "dir_hierarchy.h"
#include "filelist_io.h"
//#define DEBUG_REMOTE_DIRECTORY_STORAGE 1
/******************************************************************************************************************/
/* Iterators */
/******************************************************************************************************************/
DirectoryStorage::DirIterator::DirIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
{
mStorage = s->mFileHierarchy ;
mParentIndex = i;
mDirTabIndex = 0;
}
DirectoryStorage::FileIterator::FileIterator(DirectoryStorage *s,DirectoryStorage::EntryIndex i)
{
mStorage = s->mFileHierarchy ;
mParentIndex = i;
mFileTabIndex = 0;
}
DirectoryStorage::DirIterator& DirectoryStorage::DirIterator::operator++()
{
++mDirTabIndex ;
return *this;
}
DirectoryStorage::FileIterator& DirectoryStorage::FileIterator::operator++()
{
++mFileTabIndex ;
return *this;
}
DirectoryStorage::EntryIndex DirectoryStorage::FileIterator::operator*() const { return mStorage->getSubFileIndex(mParentIndex,mFileTabIndex) ; }
DirectoryStorage::EntryIndex DirectoryStorage::DirIterator ::operator*() const { return mStorage->getSubDirIndex(mParentIndex,mDirTabIndex) ; }
DirectoryStorage::FileIterator::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
DirectoryStorage::DirIterator ::operator bool() const { return **this != DirectoryStorage::NO_INDEX; }
RsFileHash DirectoryStorage::FileIterator::hash() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_hash):RsFileHash(); }
uint64_t DirectoryStorage::FileIterator::size() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_size):0; }
std::string DirectoryStorage::FileIterator::name() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_name):std::string(); }
time_t DirectoryStorage::FileIterator::modtime() const { const InternalFileHierarchyStorage::FileEntry *f = mStorage->getFileEntry(**this) ; return f?(f->file_modtime):0; }
std::string DirectoryStorage::DirIterator::name() const { const InternalFileHierarchyStorage::DirEntry *d = mStorage->getDirEntry(**this) ; return d?(d->dir_name):std::string(); }
/******************************************************************************************************************/
/* Directory Storage */
/******************************************************************************************************************/
DirectoryStorage::DirectoryStorage(const RsPeerId &pid)
: mPeerId(pid), mDirStorageMtx("Directory storage "+pid.toStdString())
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy = new InternalFileHierarchyStorage();
}
DirectoryStorage::EntryIndex DirectoryStorage::root() const
{
return EntryIndex(0) ;
}
int DirectoryStorage::parentRow(EntryIndex e) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->parentRow(e) ;
}
bool DirectoryStorage::getChildIndex(EntryIndex e,int row,EntryIndex& c) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->getChildIndex(e,row,c) ;
}
uint32_t DirectoryStorage::getEntryType(const EntryIndex& indx)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
switch(mFileHierarchy->getType(indx))
{
case InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR: return DIR_TYPE_DIR ;
case InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE: return DIR_TYPE_FILE ;
default:
return DIR_TYPE_UNKNOWN;
}
}
bool DirectoryStorage::getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); }
bool DirectoryStorage::getDirectoryRecursModTime(EntryIndex index,time_t& rec_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); }
bool DirectoryStorage::getDirectoryLocalModTime (EntryIndex index,time_t& loc_md_TS) const { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->getTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); }
bool DirectoryStorage::setDirectoryUpdateTime (EntryIndex index,time_t update_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,update_TS,&InternalFileHierarchyStorage::DirEntry::dir_update_time ); }
bool DirectoryStorage::setDirectoryRecursModTime(EntryIndex index,time_t rec_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,rec_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_most_recent_time); }
bool DirectoryStorage::setDirectoryLocalModTime (EntryIndex index,time_t loc_md_TS) { RS_STACK_MUTEX(mDirStorageMtx) ; return mFileHierarchy->setTS(index,loc_md_TS,&InternalFileHierarchyStorage::DirEntry::dir_modtime ); }
bool DirectoryStorage::updateSubDirectoryList(const EntryIndex& indx,const std::map<std::string,time_t>& subdirs)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubDirectoryList(indx,subdirs) ;
locked_check() ;
return res ;
}
bool DirectoryStorage::updateSubFilesList(const EntryIndex& indx,const std::map<std::string,FileTS>& subfiles,std::map<std::string,FileTS>& new_files)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->updateSubFilesList(indx,subfiles,new_files) ;
locked_check() ;
return res ;
}
bool DirectoryStorage::removeDirectory(const EntryIndex& indx)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
bool res = mFileHierarchy->removeDirectory(indx);
locked_check();
return res ;
}
void DirectoryStorage::locked_check()
{
std::string error ;
if(!mFileHierarchy->check(error))
std::cerr << "Check error: " << error << std::endl;
}
bool DirectoryStorage::updateFile(const EntryIndex& index,const RsFileHash& hash,const std::string& fname, uint64_t size,time_t modf_time)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->updateFile(index,hash,fname,size,modf_time);
}
bool DirectoryStorage::updateHash(const EntryIndex& index,const RsFileHash& hash)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->updateHash(index,hash);
}
int DirectoryStorage::searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchHash(hash,results);
}
void DirectoryStorage::load(const std::string& local_file_name)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->load(local_file_name);
}
void DirectoryStorage::save(const std::string& local_file_name)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->save(local_file_name);
}
void DirectoryStorage::print()
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mFileHierarchy->print();
}
int DirectoryStorage::searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchTerms(terms,results);
}
int DirectoryStorage::searchBoolExp(RsRegularExpression::Expression * exp, std::list<EntryIndex> &results) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->searchBoolExp(exp,results);
}
bool DirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
d.children.clear() ;
time_t now = time(NULL) ;
uint32_t type = mFileHierarchy->getType(indx) ;
d.ref = (void*)(intptr_t)indx ;
if (type == InternalFileHierarchyStorage::FileStorageNode::TYPE_DIR) /* has children --- fill */
{
const InternalFileHierarchyStorage::DirEntry *dir_entry = mFileHierarchy->getDirEntry(indx) ;
/* extract all the entries */
for(DirectoryStorage::DirIterator it(this,indx);it;++it)
{
DirStub stub;
stub.type = DIR_TYPE_DIR;
stub.name = it.name();
stub.ref = (void*)(intptr_t)*it; // this is updated by the caller, who knows which friend we're dealing with
d.children.push_back(stub);
}
for(DirectoryStorage::FileIterator it(this,indx);it;++it)
{
DirStub stub;
stub.type = DIR_TYPE_FILE;
stub.name = it.name();
stub.ref = (void*)(intptr_t)*it;
d.children.push_back(stub);
}
d.type = DIR_TYPE_DIR;
d.hash.clear() ;
d.count = dir_entry->subdirs.size() + dir_entry->subfiles.size();
d.min_age = now - dir_entry->dir_most_recent_time ;
d.name = dir_entry->dir_name;
d.path = dir_entry->dir_parent_path + "/" + dir_entry->dir_name ;
d.parent = (void*)(intptr_t)dir_entry->parent_index ;
if(indx == 0)
{
d.type = DIR_TYPE_PERSON ;
d.name = mPeerId.toStdString();
}
}
else if(type == InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)
{
const InternalFileHierarchyStorage::FileEntry *file_entry = mFileHierarchy->getFileEntry(indx) ;
d.type = DIR_TYPE_FILE;
d.count = file_entry->file_size;
d.min_age = now - file_entry->file_modtime ;
d.name = file_entry->file_name;
d.hash = file_entry->file_hash;
d.age = now - file_entry->file_modtime;
d.parent = (void*)(intptr_t)file_entry->parent_index ;
const InternalFileHierarchyStorage::DirEntry *parent_dir_entry = mFileHierarchy->getDirEntry(file_entry->parent_index);
if(parent_dir_entry != NULL)
d.path = parent_dir_entry->dir_parent_path + "/" + parent_dir_entry->dir_name + "/" ;
else
d.path = "" ;
}
else
return false;
d.flags.clear() ;
return true;
}
bool DirectoryStorage::getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->getDirHashFromIndex(index,hash) ;
}
bool DirectoryStorage::getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return mFileHierarchy->getIndexFromDirHash(hash,index) ;
}
/******************************************************************************************************************/
/* Local Directory Storage */
/******************************************************************************************************************/
void LocalDirectoryStorage::setSharedDirectoryList(const std::list<SharedDirInfo>& lst)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
// Chose virtual name if not supplied, and remove duplicates.
std::set<std::string> virtual_names ; // maps virtual to real name
std::list<SharedDirInfo> processed_list ;
for(std::list<SharedDirInfo>::const_iterator it(lst.begin());it!= lst.end();++it)
{
int i=0;
std::string candidate_virtual_name = it->virtualname ;
if(candidate_virtual_name.empty())
candidate_virtual_name = RsDirUtil::getTopDir(it->filename);
while(virtual_names.find(candidate_virtual_name) != virtual_names.end())
rs_sprintf_append(candidate_virtual_name, "-%d", ++i);
SharedDirInfo d(*it);
d.virtualname = candidate_virtual_name ;
processed_list.push_back(d) ;
virtual_names.insert(candidate_virtual_name) ;
}
mLocalDirs.clear();
for(std::list<SharedDirInfo>::const_iterator it(processed_list.begin());it!=processed_list.end();++it)
mLocalDirs[it->filename] = *it;
mTSChanged = true ;
}
void LocalDirectoryStorage::getSharedDirectoryList(std::list<SharedDirInfo>& lst)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
lst.clear();
for(std::map<std::string,SharedDirInfo>::iterator it(mLocalDirs.begin());it!=mLocalDirs.end();++it)
lst.push_back(it->second) ;
}
static bool sameLists(const std::list<RsNodeGroupId>& l1,const std::list<RsNodeGroupId>& l2)
{
std::list<RsNodeGroupId>::const_iterator it1(l1.begin()) ;
std::list<RsNodeGroupId>::const_iterator it2(l2.begin()) ;
for(; (it1!=l1.end() && it2!=l2.end());++it1,++it2)
if(*it1 != *it2)
return false ;
return it1 == l1.end() && it2 == l2.end() ;
}
void LocalDirectoryStorage::updateShareFlags(const SharedDirInfo& info)
{
bool changed = false ;
{
RS_STACK_MUTEX(mDirStorageMtx) ;
std::map<std::string,SharedDirInfo>::iterator it = mLocalDirs.find(info.filename) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) LocalDirectoryStorage::updateShareFlags: directory \"" << info.filename << "\" not found" << std::endl;
return ;
}
// we compare the new info with the old one. If the two group lists have a different order, they will be seen as different. Not a big deal. We just
// want to make sure that if they are different, flags get updated.
if(!sameLists(it->second.parent_groups,info.parent_groups) || it->second.filename != info.filename || it->second.shareflags != info.shareflags || it->second.virtualname != info.virtualname)
{
it->second = info;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Updating dir mod time because flags at level 0 have changed." << std::endl;
#endif
changed = true ;
}
}
if(changed)
{
setDirectoryLocalModTime(0,time(NULL)) ;
mTSChanged = true ;
}
}
bool LocalDirectoryStorage::convertSharedFilePath(const std::string& path, std::string& fullpath)
{
std::string shpath = RsDirUtil::removeRootDir(path);
std::string basedir = RsDirUtil::getRootDir(path);
std::string realroot ;
{
RS_STACK_MUTEX(mDirStorageMtx) ;
realroot = locked_findRealRootFromVirtualFilename(basedir);
}
if (realroot.empty())
return false;
/* construct full name */
fullpath = realroot + "/";
fullpath += shpath;
return true;
}
void LocalDirectoryStorage::notifyTSChanged()
{
RS_STACK_MUTEX(mDirStorageMtx) ;
mTSChanged = true ;
}
void LocalDirectoryStorage::updateTimeStamps()
{
RS_STACK_MUTEX(mDirStorageMtx) ;
if(mTSChanged)
{
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Updating recursive TS for local shared dirs..." << std::endl;
#endif
time_t last_modf_time = mFileHierarchy->recursUpdateLastModfTime(EntryIndex(0)) ;
mTSChanged = false ;
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "LocalDirectoryStorage: global last modf time is " << last_modf_time << " (which is " << time(NULL) - last_modf_time << " secs ago)" << std::endl;
#endif
}
}
std::string LocalDirectoryStorage::locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const
{
/**** MUST ALREADY BE LOCKED ****/
std::map<std::string, SharedDirInfo>::const_iterator cit = mLocalDirs.find(virtual_rootdir) ;
if (cit == mLocalDirs.end())
{
std::cerr << "(EE) locked_findRealRootFromVirtualFilename() Invalid RootDir: " << virtual_rootdir << std::endl;
return std::string();
}
return cit->second.filename;
}
bool LocalDirectoryStorage::extractData(const EntryIndex& indx,DirDetails& d)
{
bool res = DirectoryStorage::extractData(indx,d) ;
if(!res)
return false;
// here we should update the file sharing flags
return getFileSharingPermissions(indx,d.flags,d.parent_groups) ;
}
bool LocalDirectoryStorage::getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info)
{
DirDetails d;
extractData(i,d) ;
if(d.type != DIR_TYPE_FILE)
{
std::cerr << "(EE) LocalDirectoryStorage: asked for file info for index " << i << " which is not a file." << std::endl;
return false;
}
info.storage_permission_flags = d.flags; // Combination of the four RS_DIR_FLAGS_*. Updated when the file is a local stored file.
info.parent_groups = d.parent_groups;
info.transfer_info_flags = TransferRequestFlags(); // various flags from RS_FILE_HINTS_*
info.path = d.path + "/" + d.name;
info.fname = d.name;
info.hash = d.hash;
info.size = d.count;
// all this stuff below is not useful in this case.
info.mId = 0; /* (GUI) Model Id -> unique number */
info.ext.clear();
info.avail = 0; /* how much we have */
info.rank = 0;
info.age = 0;
info.queue_position =0;
info.searchId = 0; /* 0 if none */
/* Transfer Stuff */
info.transfered = 0;
info.tfRate = 0; /* in kbytes */
info.downloadStatus = FT_STATE_COMPLETE ;
std::list<TransferInfo> peers;
info.priority = SPEED_NORMAL;
info.lastTS = 0;
return true;
}
bool LocalDirectoryStorage::getFileSharingPermissions(const EntryIndex& indx,FileStorageFlags& flags,std::list<RsNodeGroupId>& parent_groups)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
return locked_getFileSharingPermissions(indx,flags,parent_groups) ;
}
bool LocalDirectoryStorage::locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags& flags, std::list<RsNodeGroupId> &parent_groups)
{
flags.clear() ;
parent_groups.clear();
std::string base_dir;
const InternalFileHierarchyStorage::FileStorageNode *n = mFileHierarchy->getNode(indx) ;
if(n == NULL)
return false ;
for(DirectoryStorage::EntryIndex i=((n->type()==InternalFileHierarchyStorage::FileStorageNode::TYPE_FILE)?((intptr_t)n->parent_index):indx);;)
{
const InternalFileHierarchyStorage::DirEntry *e = mFileHierarchy->getDirEntry(i) ;
if(e == NULL)
break ;
if(e->parent_index == 0)
{
base_dir = e->dir_name ;
break ;
}
i = e->parent_index ;
}
if(!base_dir.empty())
{
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(base_dir) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) very weird bug: base directory \"" << base_dir << "\" not found in shared dir list." << std::endl;
return false ;
}
flags = it->second.shareflags;
parent_groups = it->second.parent_groups;
}
return true;
}
std::string LocalDirectoryStorage::locked_getVirtualDirName(EntryIndex indx) const
{
if(indx == 0)
return std::string() ;
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
if(dir->parent_index != 0)
return dir->dir_name ;
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(dir->dir_name) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl;
return std::string() ;
}
return it->second.virtualname ;
}
std::string LocalDirectoryStorage::locked_getVirtualPath(EntryIndex indx) const
{
if(indx == 0)
return std::string() ;
std::string res ;
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
while(dir->parent_index != 0)
{
dir = mFileHierarchy->getDirEntry(dir->parent_index) ;
res += dir->dir_name + "/"+ res ;
}
std::map<std::string,SharedDirInfo>::const_iterator it = mLocalDirs.find(dir->dir_name) ;
if(it == mLocalDirs.end())
{
std::cerr << "(EE) Cannot find real name " << dir->dir_name << " at level 1 among shared dirs. Bug?" << std::endl;
return std::string() ;
}
return it->second.virtualname + "/" + res;
}
bool LocalDirectoryStorage::serialiseDirEntry(const EntryIndex& indx,RsTlvBinaryData& bindata,const RsPeerId& client_id)
{
RS_STACK_MUTEX(mDirStorageMtx) ;
const InternalFileHierarchyStorage::DirEntry *dir = mFileHierarchy->getDirEntry(indx);
if(dir == NULL)
{
std::cerr << "(EE) serialiseDirEntry: ERROR. Cannot find entry " << (void*)(intptr_t)indx << std::endl;
return false;
}
// compute list of allowed subdirs
std::vector<RsFileHash> allowed_subdirs ;
FileStorageFlags node_flags ;
std::list<RsNodeGroupId> node_groups ;
// for each subdir, compute the node flags and groups, then ask rsPeers to compute the mask that result from these flags for the particular peer supplied in parameter
for(uint32_t i=0;i<dir->subdirs.size();++i)
if(indx != 0 || (locked_getFileSharingPermissions(dir->subdirs[i],node_flags,node_groups) && (rsPeers->computePeerPermissionFlags(client_id,node_flags,node_groups) & RS_FILE_HINTS_BROWSABLE)))
{
RsFileHash hash ;
if(!mFileHierarchy->getDirHashFromIndex(dir->subdirs[i],hash))
{
std::cerr << "(EE) Cannot get hash from subdir index " << dir->subdirs[i] << ". Weird bug." << std::endl ;
return false;
}
allowed_subdirs.push_back(hash) ;
}
unsigned char *section_data = NULL;
uint32_t section_size = 0;
uint32_t section_offset = 0;
// we need to send:
// - the name of the directory, its TS
// - the index entry for each subdir (the updte TS are exchanged at a higher level)
// - the file info for each subfile
//
std::string virtual_dir_name = locked_getVirtualDirName(indx) ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,virtual_dir_name )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,(uint32_t)dir->dir_most_recent_time)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)dir->dir_modtime )) return false ;
// serialise number of subdirs and number of subfiles
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)allowed_subdirs.size() )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,(uint32_t)dir->subfiles.size() )) return false ;
// serialise subdirs entry indexes
for(uint32_t i=0;i<allowed_subdirs.size();++i)
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,allowed_subdirs[i] )) return false ;
// serialise directory subfiles, with info for each of them
for(uint32_t i=0;i<dir->subfiles.size();++i)
{
unsigned char *file_section_data = NULL ;
uint32_t file_section_offset = 0 ;
uint32_t file_section_size = 0;
const InternalFileHierarchyStorage::FileEntry *file = mFileHierarchy->getFileEntry(dir->subfiles[i]) ;
if(file == NULL)
{
std::cerr << "(EE) cannot reach file entry " << dir->subfiles[i] << " to get/send file info." << std::endl;
continue ;
}
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,file->file_name )) return false ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,file->file_size )) return false ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,file->file_hash )) return false ;
if(!FileListIO::writeField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,(uint32_t)file->file_modtime)) return false ;
// now write the whole string into a single section in the file
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_offset)) return false ;
free(file_section_data) ;
}
#ifdef DEBUG_LOCAL_DIRECTORY_STORAGE
std::cerr << "Serialised dir entry to send for entry index " << (void*)(intptr_t)indx << ". Data size is " << section_size << " bytes" << std::endl;
#endif
bindata.bin_data = section_data ;
bindata.bin_len = section_offset ;
return true ;
}
/******************************************************************************************************************/
/* Remote Directory Storage */
/******************************************************************************************************************/
RemoteDirectoryStorage::RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname)
: DirectoryStorage(pid),mLastSavedTime(0),mChanged(false),mFileName(fname)
{
load(fname) ;
std::cerr << "Loaded remote directory for peer " << pid << std::endl;
#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE
mFileHierarchy->print();
#endif
}
void RemoteDirectoryStorage::checkSave()
{
time_t now = time(NULL);
if(mChanged && mLastSavedTime + MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE < now)
{
save(mFileName);
mLastSavedTime = now ;
}
}
bool RemoteDirectoryStorage::deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& bindata)
{
const unsigned char *section_data = (unsigned char*)bindata.bin_data ;
uint32_t section_size = bindata.bin_len ;
uint32_t section_offset=0 ;
#ifdef DEBUG_REMOTE_DIRECTORY_STORAGE
std::cerr << "RemoteDirectoryStorage::deserialiseDirEntry(): deserialising directory content for friend " << peerId() << ", and directory " << indx << std::endl;
#endif
std::string dir_name ;
uint32_t most_recent_time ,dir_modtime ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_DIR_NAME ,dir_name )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RECURS_MODIF_TS,most_recent_time)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,dir_modtime )) return false ;
std::cerr << " dir name : \"" << dir_name << "\"" << std::endl;
std::cerr << " most recent time : " << most_recent_time << std::endl;
std::cerr << " modification time : " << dir_modtime << std::endl;
// serialise number of subdirs and number of subfiles
uint32_t n_subdirs,n_subfiles ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subdirs )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_RAW_NUMBER,n_subfiles )) return false ;
std::cerr << " number of subdirs : " << n_subdirs << std::endl;
std::cerr << " number of files : " << n_subfiles << std::endl;
// serialise subdirs entry indexes
std::vector<RsFileHash> subdirs_hashes ;
RsFileHash subdir_hash ;
for(uint32_t i=0;i<n_subdirs;++i)
{
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_ENTRY_INDEX ,subdir_hash)) return false ;
subdirs_hashes.push_back(subdir_hash) ;
}
// deserialise directory subfiles, with info for each of them
std::vector<InternalFileHierarchyStorage::FileEntry> subfiles_array ;
for(uint32_t i=0;i<n_subfiles;++i)
{
// Read the full data section for the file
unsigned char *file_section_data = NULL ;
uint32_t file_section_size = 0;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY,file_section_data,file_section_size)) return false ;
uint32_t file_section_offset = 0 ;
InternalFileHierarchyStorage::FileEntry f;
uint32_t modtime =0;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_NAME ,f.file_name )) return false ;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,f.file_size )) return false ;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,f.file_hash )) return false ;
if(!FileListIO::readField(file_section_data,file_section_size,file_section_offset,FILE_LIST_IO_TAG_MODIF_TS ,modtime )) return false ;
f.file_modtime = modtime ;
free(file_section_data) ;
subfiles_array.push_back(f) ;
}
RS_STACK_MUTEX(mDirStorageMtx) ;
std::cerr << " updating dir entry..." << std::endl;
// First create the entries for each subdir and each subfile, if needed.
if(!mFileHierarchy->updateDirEntry(indx,dir_name,most_recent_time,dir_modtime,subdirs_hashes,subfiles_array))
{
std::cerr << "(EE) Cannot update dir entry with index " << indx << ": entry does not exist." << std::endl;
return false ;
}
mChanged = true ;
return true ;
}

285
libretroshare/src/file_sharing/directory_storage.h Normal file
View File

@ -0,0 +1,285 @@
/*
* RetroShare C++ Directory Storage system.
*
* file_sharing/directory_storage.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <string>
#include <stdint.h>
#include <list>
#include "retroshare/rsids.h"
#include "retroshare/rsfiles.h"
#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
class RsTlvBinaryData ;
class InternalFileHierarchyStorage ;
class RsTlvBinaryData ;
class DirectoryStorage
{
public:
DirectoryStorage(const RsPeerId& pid) ;
virtual ~DirectoryStorage() {}
typedef uint32_t EntryIndex ;
static const EntryIndex NO_INDEX = 0xffffffff;
void save() const ;
// These functions are to be used by file transfer and file search.
virtual int searchTerms(const std::list<std::string>& terms, std::list<EntryIndex> &results) const ;
virtual int searchBoolExp(RsRegularExpression::Expression * exp, std::list<EntryIndex> &results) const ;
virtual int searchHash(const RsFileHash& hash, std::list<EntryIndex> &results) const ;
// gets/sets the various time stamps:
//
bool getDirectoryRecursModTime(EntryIndex index,time_t& recurs_max_modf_TS) const ; // last modification time, computed recursively over all subfiles and directories
bool getDirectoryLocalModTime (EntryIndex index,time_t& motime_TS) const ; // last modification time for that index only
bool getDirectoryUpdateTime (EntryIndex index,time_t& update_TS) const ; // last time the entry was updated. This is only used on the RemoteDirectoryStorage side.
bool setDirectoryRecursModTime(EntryIndex index,time_t recurs_max_modf_TS) ;
bool setDirectoryLocalModTime (EntryIndex index,time_t modtime_TS) ;
bool setDirectoryUpdateTime (EntryIndex index,time_t update_TS) ;
uint32_t getEntryType(const EntryIndex& indx) ; // WARNING: returns DIR_TYPE_*, not the internal directory storage stuff.
virtual bool extractData(const EntryIndex& indx,DirDetails& d);
// This class allows to abstractly browse the stored directory hierarchy in a depth-first manner.
// It gives access to sub-files and sub-directories below. When using it, the client should make sure
// that the DirectoryStorage is properly locked, since the iterator cannot lock it.
//
class DirIterator
{
public:
DirIterator(const DirIterator& d) ;
DirIterator(DirectoryStorage *d,EntryIndex i) ;
DirIterator& operator++() ;
EntryIndex operator*() const ;
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
// info about the directory that is pointed by the iterator
std::string name() const ;
time_t last_modif_time() const ; // last time a file in this directory or in the directories below has been modified.
time_t last_update_time() const ; // last time this directory was updated
private:
EntryIndex mParentIndex ; // index of the parent dir.
uint32_t mDirTabIndex ; // index in the vector of subdirs.
InternalFileHierarchyStorage *mStorage ;
friend class DirectoryStorage ;
};
class FileIterator
{
public:
FileIterator(DirIterator& d); // crawls all files in specified directory
FileIterator(DirectoryStorage *d,EntryIndex e); // crawls all files in specified directory
FileIterator& operator++() ;
EntryIndex operator*() const ; // current file entry
operator bool() const ; // used in for loops. Returns true when the iterator is valid.
// info about the file that is pointed by the iterator
std::string name() const ;
uint64_t size() const ;
RsFileHash hash() const ;
time_t modtime() const ;
private:
EntryIndex mParentIndex ; // index of the parent dir.
uint32_t mFileTabIndex ; // index in the vector of subdirs.
InternalFileHierarchyStorage *mStorage ;
};
struct FileTS
{
uint64_t size ;
time_t modtime;
};
EntryIndex root() const ; // returns the index of the root directory entry. This is generally 0.
const RsPeerId& peerId() const { return mPeerId ; } // peer ID of who owns that file list.
int parentRow(EntryIndex e) const ; // position of the current node, in the array of children at its parent node. Used by GUI for display.
bool getChildIndex(EntryIndex e,int row,EntryIndex& c) const; // returns the index of the children node at position "row" in the children nodes. Used by GUI for display.
// Sets the subdirectory/subfiles list of entry indx the supplied one, possible adding and removing directories (resp.files). New directories are set empty with
// just a name and need to be updated later on. New files are returned in a list so that they can be sent to hash cache.
//
bool updateSubDirectoryList(const EntryIndex& indx, const std::map<std::string, time_t> &subdirs) ;
bool updateSubFilesList(const EntryIndex& indx, const std::map<std::string, FileTS> &subfiles, std::map<std::string, FileTS> &new_files) ;
bool removeDirectory(const EntryIndex& indx) ;
// Updates relevant information for the file at the given index.
bool updateFile(const EntryIndex& index,const RsFileHash& hash, const std::string& fname, uint64_t size, time_t modf_time) ;
bool updateHash(const EntryIndex& index,const RsFileHash& hash);
// Returns the hash of the directory at the given index and reverse. This hash is set as random the first time it is used (when updating directories). It will be
// used by the sync system to designate the directory without referring to index (index could be used to figure out the existance of hidden directories)
bool getDirHashFromIndex(const EntryIndex& index,RsFileHash& hash) const ; // constant cost
bool getIndexFromDirHash(const RsFileHash& hash,EntryIndex& index) const ; // log cost.
void print();
void cleanup();
protected:
void load(const std::string& local_file_name) ;
void save(const std::string& local_file_name) ;
private:
// debug
void locked_check();
// storage of internal structure. Totally hidden from the outside. EntryIndex is simply the index of the entry in the vector.
RsPeerId mPeerId;
protected:
mutable RsMutex mDirStorageMtx ;
InternalFileHierarchyStorage *mFileHierarchy ;
};
class RemoteDirectoryStorage: public DirectoryStorage
{
public:
RemoteDirectoryStorage(const RsPeerId& pid,const std::string& fname) ;
virtual ~RemoteDirectoryStorage() {}
/*!
* \brief deserialiseDirEntry
* Loads a serialised directory content coming from a friend. The directory entry needs to exist already,
* as it is created when updating the parent.
*
* \param indx index of the directory to update
* \param bindata binary data to deserialise from
* \return false when the directory cannot be found.
*/
bool deserialiseUpdateDirEntry(const EntryIndex& indx,const RsTlvBinaryData& data) ;
/*!
* \brief checkSave
* Checks the time of last saving, last modification time, and saves if needed.
*/
void checkSave() ;
private:
time_t mLastSavedTime ;
bool mChanged ;
std::string mFileName;
};
class LocalDirectoryStorage: public DirectoryStorage
{
public:
LocalDirectoryStorage(const std::string& fname,const RsPeerId& own_id) : DirectoryStorage(own_id),mFileName(fname) {}
virtual ~LocalDirectoryStorage() {}
/*!
* \brief [gs]etSharedDirectoryList
* Gets/sets the list of shared directories. Each directory is supplied with a virtual name (the name the friends will see), and sharing flags/groups.
* \param lst
*/
void setSharedDirectoryList(const std::list<SharedDirInfo>& lst) ;
void getSharedDirectoryList(std::list<SharedDirInfo>& lst) ;
void updateShareFlags(const SharedDirInfo& info) ;
bool convertSharedFilePath(const std::string& path_with_virtual_name,std::string& fullpath) ;
/*!
* \brief updateTimeStamps
* Checks recursive TS and update the if needed.
*/
void updateTimeStamps();
/*!
* \brief notifyTSChanged
* Use this to force an update of the recursive TS, when calling updateTimeStamps();
*/
void notifyTSChanged();
/*!
* \brief getFileInfo Converts an index info a full file info structure.
* \param i index in the directory structure
* \param info structure to be filled in
* \return false if the file does not exist, or is a directory,...
*/
bool getFileInfo(DirectoryStorage::EntryIndex i,FileInfo& info) ;
/*!
* \brief getFileSharingPermissions
* Computes the flags and parent groups for any index.
* \param indx index of the entry to compute the flags for
* \param flags computed flags
* \param parent_groups computed parent groups
* \return
* false if the index is not valid
* false otherwise
*/
bool getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list<RsNodeGroupId> &parent_groups);
virtual bool extractData(const EntryIndex& indx,DirDetails& d) ;
/*!
* \brief serialiseDirEntry
* Produced a serialised directory content listing suitable for export to friends.
*
* \param indx index of the directory to serialise
* \param bindata binary data created by serialisation
* \param client_id Peer id to be serialised to. Depending on permissions, some subdirs can be removed.
* \return false when the directory cannot be found.
*/
bool serialiseDirEntry(const EntryIndex& indx, RsTlvBinaryData& bindata, const RsPeerId &client_id) ;
private:
std::string locked_getVirtualPath(EntryIndex indx) const ;
std::string locked_getVirtualDirName(EntryIndex indx) const ;
bool locked_getFileSharingPermissions(const EntryIndex& indx, FileStorageFlags &flags, std::list<RsNodeGroupId>& parent_groups);
std::string locked_findRealRootFromVirtualFilename(const std::string& virtual_rootdir) const;
std::map<std::string,SharedDirInfo> mLocalDirs ; // map is better for search. it->first=it->second.filename
std::string mFileName;
bool mTSChanged ;
};

230
libretroshare/src/file_sharing/directory_updater.cc Normal file
View File

@ -0,0 +1,230 @@
/*
* RetroShare Directory watching system.
*
* file_sharing/directory_updater.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "util/folderiterator.h"
#include "rsserver/p3face.h"
#include "directory_storage.h"
#include "directory_updater.h"
#include "file_sharing_defaults.h"
//#define DEBUG_LOCAL_DIR_UPDATER 1
//=============================================================================================================//
// Local Directory Updater //
//=============================================================================================================//
LocalDirectoryUpdater::LocalDirectoryUpdater(HashStorage *hc,LocalDirectoryStorage *lds)
: mHashCache(hc),mSharedDirectories(lds)
{
mLastSweepTime = 0;
mLastTSUpdateTime = 0;
mDelayBetweenDirectoryUpdates = DELAY_BETWEEN_DIRECTORY_UPDATES;
mIsEnabled = false ;
}
bool LocalDirectoryUpdater::isEnabled() const
{
return mIsEnabled ;
}
void LocalDirectoryUpdater::setEnabled(bool b)
{
if(mIsEnabled == b)
return ;
if(b)
start() ;
else
shutdown();
mIsEnabled = b ;
}
void LocalDirectoryUpdater::data_tick()
{
time_t now = time(NULL) ;
if(now > mDelayBetweenDirectoryUpdates + mLastSweepTime)
{
sweepSharedDirectories() ;
mLastSweepTime = now;
mSharedDirectories->notifyTSChanged() ;
}
if(now > DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE + mLastTSUpdateTime)
{
mSharedDirectories->updateTimeStamps() ;
mLastTSUpdateTime = now ;
}
usleep(10*1000*1000);
}
void LocalDirectoryUpdater::forceUpdate()
{
mLastSweepTime = 0;
}
void LocalDirectoryUpdater::sweepSharedDirectories()
{
RsServer::notify()->notifyListPreChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] LocalDirectoryUpdater::sweep()" << std::endl;
#endif
// recursive update algorithm works that way:
// - the external loop starts on the shared directory list and goes through sub-directories
// - at the same time, it updates the local list of shared directories. A single sweep is performed over the whole directory structure.
// - the information that is costly to compute (the hash) is store externally into a separate structure.
// - doing so, changing directory names or moving files between directories does not cause a re-hash of the content.
//
std::list<SharedDirInfo> shared_directory_list ;
mSharedDirectories->getSharedDirectoryList(shared_directory_list);
std::map<std::string,time_t> sub_dir_list ;
for(std::list<SharedDirInfo>::const_iterator real_dir_it(shared_directory_list.begin());real_dir_it!=shared_directory_list.end();++real_dir_it)
sub_dir_list[(*real_dir_it).filename] = 0 ;
// make sure that entries in stored_dir_it are the same than paths in real_dir_it, and in the same order.
mSharedDirectories->updateSubDirectoryList(mSharedDirectories->root(),sub_dir_list) ;
// now for each of them, go recursively and match both files and dirs
for(DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,mSharedDirectories->root()) ; stored_dir_it;++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(stored_dir_it.name(), *stored_dir_it) ; // here we need to use the list that was stored, instead of the shared dir list, because the two
// are not necessarily in the same order.
}
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
}
void LocalDirectoryUpdater::recursUpdateSharedDir(const std::string& cumulated_path, DirectoryStorage::EntryIndex indx)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << "[directory storage] parsing directory " << cumulated_path << ", index=" << indx << std::endl;
#endif
// make sure list of subdirs is the same
// make sure list of subfiles is the same
// request all hashes to the hashcache
librs::util::FolderIterator dirIt(cumulated_path);
// collect subdirs and subfiles
std::map<std::string,DirectoryStorage::FileTS> subfiles ;
std::map<std::string,time_t> subdirs ;
for(;dirIt.isValid();dirIt.next())
{
switch(dirIt.file_type())
{
case librs::util::FolderIterator::TYPE_FILE: subfiles[dirIt.file_name()].modtime = dirIt.file_modtime() ;
subfiles[dirIt.file_name()].size = dirIt.file_size();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-file \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
case librs::util::FolderIterator::TYPE_DIR: subdirs[dirIt.file_name()] = dirIt.file_modtime();
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " adding sub-dir \"" << dirIt.file_name() << "\"" << std::endl;
#endif
break;
default:
std::cerr << "(EE) Dir entry of unknown type with path \"" << cumulated_path << "/" << dirIt.file_name() << "\"" << std::endl;
}
}
// update folder modificatoin time, which is the only way to detect e.g. removed or renamed files.
mSharedDirectories->setDirectoryLocalModTime(indx,dirIt.dir_modtime()) ;
// update file and dir lists for current directory.
mSharedDirectories->updateSubDirectoryList(indx,subdirs) ;
std::map<std::string,DirectoryStorage::FileTS> new_files ;
mSharedDirectories->updateSubFilesList(indx,subfiles,new_files) ;
// now go through list of subfiles and request the hash to hashcache
for(DirectoryStorage::FileIterator dit(mSharedDirectories,indx);dit;++dit)
{
// ask about the hash. If not present, ask HashCache. If not present, or different, the callback will update it.
RsFileHash hash ;
if(mHashCache->requestHash(cumulated_path + "/" + dit.name(),dit.size(),dit.modtime(),hash,this,*dit) && dit.hash() != hash)
mSharedDirectories->updateHash(*dit,hash);
}
// go through the list of sub-dirs and recursively update
DirectoryStorage::DirIterator stored_dir_it(mSharedDirectories,indx) ;
for(std::map<std::string,time_t>::const_iterator real_dir_it(subdirs.begin());real_dir_it!=subdirs.end();++real_dir_it, ++stored_dir_it)
{
#ifdef DEBUG_LOCAL_DIR_UPDATER
std::cerr << " recursing into " << stored_dir_it.name() << std::endl;
#endif
recursUpdateSharedDir(cumulated_path + "/" + stored_dir_it.name(), *stored_dir_it) ;
}
}
bool LocalDirectoryUpdater::inDirectoryCheck() const
{
return mHashCache->isRunning();
}
void LocalDirectoryUpdater::hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)
{
if(!mSharedDirectories->updateHash(DirectoryStorage::EntryIndex(client_param),hash))
std::cerr << "(EE) Cannot update file. Something's wrong." << std::endl;
mSharedDirectories->notifyTSChanged() ;
}
bool LocalDirectoryUpdater::hash_confirm(uint32_t client_param)
{
return mSharedDirectories->getEntryType(DirectoryStorage::EntryIndex(client_param)) == DIR_TYPE_FILE ;
}
void LocalDirectoryUpdater::setFileWatchPeriod(int seconds)
{
mDelayBetweenDirectoryUpdates = seconds ;
}
uint32_t LocalDirectoryUpdater::fileWatchPeriod() const
{
return mDelayBetweenDirectoryUpdates ;
}

69
libretroshare/src/file_sharing/directory_updater.h Normal file
View File

@ -0,0 +1,69 @@
/*
* RetroShare C++ Directory parsing code.
*
* file_sharing/directory_updater.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
// This class crawls the given directry hierarchy and updates it. It does so by calling the
// shared file list source. This source may be of two types:
// - local: directories are crawled n disk and files are hashed / requested from a cache
// - remote: directories are requested remotely to a providing client
//
#include "file_sharing/hash_cache.h"
#include "file_sharing/directory_storage.h"
class LocalDirectoryUpdater: public HashStorageClient, public RsTickingThread
{
public:
LocalDirectoryUpdater(HashStorage *hash_cache,LocalDirectoryStorage *lds) ;
virtual ~LocalDirectoryUpdater() {}
void forceUpdate();
bool inDirectoryCheck() const ;
void setFileWatchPeriod(int seconds) ;
uint32_t fileWatchPeriod() const ;
void setEnabled(bool b) ;
bool isEnabled() const ;
protected:
virtual void data_tick() ;
virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size);
virtual bool hash_confirm(uint32_t client_param) ;
void recursUpdateSharedDir(const std::string& cumulated_path,DirectoryStorage::EntryIndex indx);
void sweepSharedDirectories();
private:
HashStorage *mHashCache ;
LocalDirectoryStorage *mSharedDirectories ;
time_t mLastSweepTime;
time_t mLastTSUpdateTime;
uint32_t mDelayBetweenDirectoryUpdates;
bool mIsEnabled ;
};

44
libretroshare/src/file_sharing/file_sharing_defaults.h Normal file
View File

@ -0,0 +1,44 @@
/*
* RetroShare C++ File sharing default variables
*
* file_sharing/file_sharing_defaults.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes
static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 sec. Buy we only update for real if something has changed.
static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time
static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.
static const uint32_t MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE = 20 ; // never save hash cache more often than every 20 secs.
static const uint32_t MIN_INTERVAL_BETWEEN_REMOTE_DIRECTORY_SAVE = 23 ; // never save remote directories more often than this
static const uint32_t MAX_DIR_SYNC_RESPONSE_DATA_SIZE = 20000 ; // Maximum RsItem data size in bytes for serialised directory transmission
static const uint32_t DEFAULT_HASH_STORAGE_DURATION_DAYS = 30 ; // remember deleted/inaccessible files for 30 days

232
libretroshare/src/file_sharing/filelist_io.cc Normal file
View File

@ -0,0 +1,232 @@
/*
* RetroShare File lists IO methods.
*
* file_sharing/filelist_io.h
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "retroshare/rsids.h"
#include "pqi/authssl.h"
#include "util/rsdir.h"
#include "serialiser/rsbaseserial.h"
#include "filelist_io.h"
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint32_t & val) { return setRawUInt32(buff,size,&offset,val) ; }
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const uint64_t & val) { return setRawUInt64(buff,size,&offset,val) ; }
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const std::string & val) { return setRawString(buff,size,&offset,val) ; }
template<> bool FileListIO::serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const Sha1CheckSum & val) { return val.serialise(buff,size,offset) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint32_t & val) { return getRawUInt32(const_cast<uint8_t*>(buff),size,&offset,&val) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,uint64_t & val) { return getRawUInt64(const_cast<uint8_t*>(buff),size,&offset,&val) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,std::string & val) { return getRawString(const_cast<uint8_t*>(buff),size,&offset,val) ; }
template<> bool FileListIO::deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,Sha1CheckSum & val) { return val.deserialise(const_cast<uint8_t*>(buff),size,offset) ; }
template<> uint32_t FileListIO::serial_size(const uint32_t & ) { return 4 ; }
template<> uint32_t FileListIO::serial_size(const uint64_t & ) { return 8 ; }
template<> uint32_t FileListIO::serial_size(const std::string & val) { return getRawStringSize(val) ; }
template<> uint32_t FileListIO::serial_size(const Sha1CheckSum & ) { return Sha1CheckSum::serial_size(); }
bool FileListIO::writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size)
{
if(!checkSectionSize(buff,buff_size,offset,size))
return false;
if(!writeSectionHeader(buff,buff_size,offset,section_tag,size))
return false;
memcpy(&buff[offset],val,size) ;
offset += size ;
return true;
}
bool FileListIO::readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size)
{
if(!readSectionHeader(buff,buff_size,offset,check_section_tag,size))
return false;
val = (unsigned char *)rs_malloc(size) ;
if(!val)
return false;
memcpy(val,&buff[offset],size);
offset += size ;
return true ;
}
bool FileListIO::write125Size(unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t S)
{
if(S < 192)
{
if(offset+1 > data_size)
return false;
data[offset++] = (uint8_t)S ;
return true;
}
else if(S < 8384)
{
if(offset+2 > data_size)
return false;
data[offset+1] = (S - 192) & 0xff ;
data[offset ] = ((S - 192 - data[offset+1]) >> 8) + 192;
offset += 2 ;
return true;
}
else
{
if(offset+5 > data_size)
return false;
data[offset++] = 0xff ;
data[offset++] = (uint8_t)((S >> 24) & 255) ;
data[offset++] = (uint8_t)((S >> 16) & 255) ;
data[offset++] = (uint8_t)((S >> 8) & 255) ;
data[offset++] = (uint8_t)((S ) & 255) ;
return true ;
}
}
bool FileListIO::read125Size(const unsigned char *data,uint32_t data_size,uint32_t& offset,uint32_t& S)
{
if(offset + 1 >= data_size) return false;
uint8_t b1 = data[offset++] ;
if(b1 < 192)
{
S = b1;
return true ;
}
if(offset + 1 >= data_size) return false;
uint8_t b2 = data[offset++] ;
if(b1 < 224)
{
S = ((b1-192) << 8) + b2 + 192 ;
return true;
}
if(b1 != 0xff)
return false;
if(offset + 3 >= data_size) return false;
uint8_t b3 = data[offset++];
uint8_t b4 = data[offset++];
uint8_t b5 = data[offset++];
S = (b2 << 24) | (b3 << 16) | (b4 << 8) | b5 ;
return true;
}
bool FileListIO::saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size)
{
void *encryptedData = NULL ;
int encDataLen = 0 ;
if(!AuthSSL::getAuthSSL()->encrypt( encryptedData, encDataLen, data,total_size, AuthSSL::getAuthSSL()->OwnId()))
{
std::cerr << "Cannot encrypt hash cache. Something's wrong." << std::endl;
return false;
}
FILE *F = fopen( (fname+".tmp").c_str(),"wb" ) ;
if(!F)
{
std::cerr << "Cannot open encrypted file cache for writing: " << fname+".tmp" << std::endl;
free(encryptedData);
return false;
}
if(fwrite(encryptedData,1,encDataLen,F) != (uint32_t)encDataLen)
{
std::cerr << "Could not write entire encrypted hash cache file. Out of disc space??" << std::endl;
fclose(F) ;
free(encryptedData);
return false;
}
fclose(F) ;
RsDirUtil::renameFile(fname+".tmp",fname) ;
#ifdef FIM_DEBUG
std::cerr << "done." << std::endl ;
#endif
free(encryptedData);
return true;
}
bool FileListIO::loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size)
{
uint64_t file_size ;
if(!RsDirUtil::checkFile( fname,file_size,false ) )
{
std::cerr << "Encrypted hash cache file not present." << std::endl;
return false;
}
// read the binary stream into memory.
//
RsTemporaryMemory buffer(file_size) ;
if(buffer == NULL)
return false;
FILE *F = fopen( fname.c_str(),"rb") ;
if (!F)
{
std::cerr << "Cannot open file for reading encrypted file cache, filename " << fname << std::endl;
return false;
}
if(fread(buffer,1,file_size,F) != file_size)
{
std::cerr << "Cannot read from file " + fname << ": something's wrong." << std::endl;
fclose(F) ;
return false;
}
fclose(F) ;
// now decrypt
void *decrypted_data =NULL;
int decrypted_data_size =0;
if(!AuthSSL::getAuthSSL()->decrypt(decrypted_data, decrypted_data_size, buffer, file_size))
{
std::cerr << "Cannot decrypt encrypted file cache. Something's wrong." << std::endl;
return false;
}
data = (unsigned char*)decrypted_data ;
total_size = decrypted_data_size ;
return true;
}

145
libretroshare/src/file_sharing/filelist_io.h Normal file
View File

@ -0,0 +1,145 @@
/*
* RetroShare C++ File lists IO methods.
*
* file_sharing/filelist_io.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <stdlib.h>
#include <stdint.h>
#include <string.h>
#include "util/rsmemory.h"
// This file implements load/save of various fields used for file lists and directory content.
// WARNING: the encoding is system-dependent, so this should *not* be used to exchange data between computers.
static const uint32_t FILE_LIST_IO_LOCAL_DIRECTORY_STORAGE_VERSION_0001 = 0x00000001 ;
static const uint8_t FILE_LIST_IO_TAG_UNKNOWN = 0x00 ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIRECTORY_VERSION = 0x01 ;
static const uint8_t FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY = 0x10 ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_FILE_ENTRY = 0x11 ;
static const uint8_t FILE_LIST_IO_TAG_LOCAL_DIR_ENTRY = 0x12 ;
static const uint8_t FILE_LIST_IO_TAG_REMOTE_FILE_ENTRY = 0x13 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SHA1_HASH = 0x20 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_NAME = 0x21 ;
static const uint8_t FILE_LIST_IO_TAG_FILE_SIZE = 0x22 ;
static const uint8_t FILE_LIST_IO_TAG_MODIF_TS = 0x30 ;
static const uint8_t FILE_LIST_IO_TAG_RECURS_MODIF_TS = 0x31 ;
static const uint8_t FILE_LIST_IO_TAG_UPDATE_TS = 0x32 ;
static const uint8_t FILE_LIST_IO_TAG_ENTRY_INDEX = 0x40 ;
static const uint8_t FILE_LIST_IO_TAG_PARENT_INDEX = 0x41 ;
static const uint8_t FILE_LIST_IO_TAG_DIR_HASH = 0x50 ;
static const uint8_t FILE_LIST_IO_TAG_DIR_NAME = 0x51 ;
static const uint8_t FILE_LIST_IO_TAG_ROW = 0x60 ;
static const uint8_t FILE_LIST_IO_TAG_BINARY_DATA = 0x61 ;
static const uint8_t FILE_LIST_IO_TAG_RAW_NUMBER = 0x62 ;
static const uint32_t SECTION_HEADER_MAX_SIZE = 6 ; // section tag (1 byte) + size (max = 5 bytes)
class FileListIO
{
public:
template<typename T>
static bool writeField(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const T& val)
{
uint32_t s = serial_size(val) ;
if(!checkSectionSize(buff,buff_size,offset,s))
return false;
if(!writeSectionHeader(buff,buff_size,offset,section_tag,s))
return false;
return serialise(buff,buff_size,offset,val) ;
}
template<typename T>
static bool readField(const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,T& val)
{
uint32_t section_size ;
if(!readSectionHeader(buff,buff_size,offset,check_section_tag,section_size))
return false;
return deserialise(buff,buff_size,offset,val);
}
static bool writeField( unsigned char*&buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,const unsigned char * val,uint32_t size) ;
static bool readField (const unsigned char *buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag, unsigned char *& val,uint32_t& size) ;
template<class T> static bool serialise(unsigned char *buff,uint32_t size,uint32_t& offset,const T& val) ;
template<class T> static bool deserialise(const unsigned char *buff,uint32_t size,uint32_t& offset,T& val) ;
template<class T> static uint32_t serial_size(const T& val) ;
static bool saveEncryptedDataToFile(const std::string& fname,const unsigned char *data,uint32_t total_size);
static bool loadEncryptedDataFromFile(const std::string& fname,unsigned char *& data,uint32_t& total_size);
private:
static bool write125Size(unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t size) ;
static bool read125Size (const unsigned char *data,uint32_t total_size,uint32_t& offset,uint32_t& size) ;
static bool checkSectionSize(unsigned char *& buff,uint32_t& buff_size,uint32_t offset,uint32_t S)
{
if(offset + S + SECTION_HEADER_MAX_SIZE > buff_size)
{
buff = (unsigned char *)realloc(buff,offset + S + SECTION_HEADER_MAX_SIZE) ;
buff_size = offset + S + SECTION_HEADER_MAX_SIZE;
if(!buff)
return false ;
}
return true ;
}
static bool writeSectionHeader(unsigned char *& buff,uint32_t& buff_size,uint32_t& offset,uint8_t section_tag,uint32_t S)
{
buff[offset++] = section_tag ;
if(!write125Size(buff,buff_size,offset,S)) return false ;
return true;
}
static bool readSectionHeader(const unsigned char *& buff,uint32_t buff_size,uint32_t& offset,uint8_t check_section_tag,uint32_t& S)
{
if(offset + 1 > buff_size)
return false ;
uint8_t section_tag = buff[offset] ; // we do the offset++ after, only if the header can be read. Doing so, we can make multiple read attempts.
if(section_tag != check_section_tag)
return false;
offset++ ;
return read125Size(buff,buff_size,offset,S) ;
}
};

393
libretroshare/src/file_sharing/hash_cache.cc Normal file
View File

@ -0,0 +1,393 @@
/*
* RetroShare Hash cache
*
* file_sharing/hash_cache.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "util/rsdir.h"
#include "util/rsprint.h"
#include "rsserver/p3face.h"
#include "pqi/authssl.h"
#include "hash_cache.h"
#include "filelist_io.h"
#include "file_sharing_defaults.h"
//#define HASHSTORAGE_DEBUG 1
static const uint32_t DEFAULT_INACTIVITY_SLEEP_TIME = 50*1000;
static const uint32_t MAX_INACTIVITY_SLEEP_TIME = 2*1000*1000;
HashStorage::HashStorage(const std::string& save_file_name)
: mFilePath(save_file_name), mHashMtx("Hash Storage mutex")
{
mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME;
mRunning = false ;
mLastSaveTime = 0 ;
mTotalSizeToHash = 0;
mTotalFilesToHash = 0;
mMaxStorageDurationDays = DEFAULT_HASH_STORAGE_DURATION_DAYS ;
{
RS_STACK_MUTEX(mHashMtx) ;
locked_load() ;
}
}
static std::string friendlyUnit(uint64_t val)
{
const std::string units[5] = {"B","KB","MB","GB","TB"};
char buf[50] ;
double fact = 1.0 ;
for(unsigned int i=0; i<5; ++i)
if(double(val)/fact < 1024.0)
{
sprintf(buf,"%2.2f",double(val)/fact) ;
return std::string(buf) + " " + units[i];
}
else
fact *= 1024.0f ;
sprintf(buf,"%2.2f",double(val)/fact*1024.0f) ;
return std::string(buf) + " TB";
}
void HashStorage::data_tick()
{
FileHashJob job;
RsFileHash hash;
uint64_t size ;
{
bool empty ;
uint32_t st ;
{
RS_STACK_MUTEX(mHashMtx) ;
if(mChanged && mLastSaveTime + MIN_INTERVAL_BETWEEN_HASH_CACHE_SAVE < time(NULL))
{
locked_save();
mLastSaveTime = time(NULL) ;
mChanged = false ;
}
}
{
RS_STACK_MUTEX(mHashMtx) ;
empty = mFilesToHash.empty();
st = mInactivitySleepTime ;
}
// sleep off mutex!
if(empty)
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << "nothing to hash. Sleeping for " << st << " us" << std::endl;
#endif
usleep(st); // when no files to hash, just wait for 2 secs. This avoids a dramatic loop.
if(st > MAX_INACTIVITY_SLEEP_TIME)
{
RS_STACK_MUTEX(mHashMtx) ;
mInactivitySleepTime = MAX_INACTIVITY_SLEEP_TIME;
if(!mChanged) // otherwise it might prevent from saving the hash cache
{
std::cerr << "Stopping hashing thread." << std::endl;
shutdown();
mRunning = false ;
mTotalSizeToHash = 0;
mTotalFilesToHash = 0;
std::cerr << "done." << std::endl;
}
RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_FINISH, "") ;
}
else
{
RS_STACK_MUTEX(mHashMtx) ;
mInactivitySleepTime = 2*st ;
}
return ;
}
mInactivitySleepTime = DEFAULT_INACTIVITY_SLEEP_TIME;
{
RS_STACK_MUTEX(mHashMtx) ;
job = mFilesToHash.begin()->second ;
mFilesToHash.erase(mFilesToHash.begin()) ;
}
if(job.client->hash_confirm(job.client_param))
{
std::cerr << "Hashing file " << job.full_path << "..." ; std::cerr.flush();
std::string tmpout;
rs_sprintf(tmpout, "%lu/%lu (%s - %d%%) : %s", (unsigned long int)mHashCounter+1, (unsigned long int)mTotalFilesToHash, friendlyUnit(mTotalHashedSize).c_str(), int(mTotalHashedSize/double(mTotalSizeToHash)*100.0), job.full_path.c_str()) ;
RsServer::notify()->notifyHashingInfo(NOTIFY_HASHTYPE_HASH_FILE, tmpout) ;
if(!RsDirUtil::getFileHash(job.full_path, hash,size, this))
std::cerr << "ERROR: cannot hash file " << job.full_path << std::endl;
else
std::cerr << "done."<< std::endl;
// store the result
{
RS_STACK_MUTEX(mHashMtx) ;
HashStorageInfo& info(mFiles[job.full_path]);
info.filename = job.full_path ;
info.size = size ;
info.modf_stamp = job.ts ;
info.time_stamp = time(NULL);
info.hash = hash;
mChanged = true ;
++mHashCounter ;
mTotalHashedSize += size ;
}
}
}
// call the client
if(!hash.isNull())
job.client->hash_callback(job.client_param, job.full_path, hash, size);
}
bool HashStorage::requestHash(const std::string& full_path,uint64_t size,time_t mod_time,RsFileHash& known_hash,HashStorageClient *c,uint32_t client_param)
{
// check if the hash is up to date w.r.t. cache.
#ifdef HASHSTORAGE_DEBUG
std::cerr << "HASH Requested for file " << full_path << ": ";
#endif
RS_STACK_MUTEX(mHashMtx) ;
time_t now = time(NULL) ;
std::map<std::string,HashStorageInfo>::iterator it = mFiles.find(full_path) ;
// On windows we compare the time up to +/- 3600 seconds. This avoids re-hashing files in case of daylight saving change.
//
// See:
// https://support.microsoft.com/en-us/kb/190315
//
if(it != mFiles.end()
#ifdef WINDOWS_SYS
&& ( (uint64_t)mod_time == it->second.modf_stamp || (uint64_t)mod_time+3600 == it->second.modf_stamp ||(uint64_t)mod_time == it->second.modf_stamp+3600)
#else
&& (uint64_t)mod_time == it->second.modf_stamp
#endif
&& size == it->second.size)
{
it->second.time_stamp = now ;
#ifdef WINDOWS_SYS
if(it->second.time_stamp != (uint64_t)mod_time)
{
std::cerr << "(WW) detected a 1 hour shift in file modification time. This normally happens to many files at once, when daylight saving time shifts (file=\"" << full_path << "\")." << std::endl;
it->second.time_stamp = (uint64_t)mod_time;
}
#endif
known_hash = it->second.hash;
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Found in cache." << std::endl ;
#endif
return true ;
}
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Not in cache. Scheduling for re-hash." << std::endl ;
#endif
// we need to schedule a re-hashing
if(mFilesToHash.find(full_path) != mFilesToHash.end())
return false ;
FileHashJob job ;
job.client = c ;
job.size = size ;
job.client_param = client_param ;
job.full_path = full_path ;
job.ts = mod_time ;
mFilesToHash[full_path] = job;
mTotalSizeToHash += size ;
++mTotalFilesToHash;
if(!mRunning)
{
mRunning = true ;
std::cerr << "Starting hashing thread." << std::endl;
mHashCounter = 0;
mTotalHashedSize = 0;
start() ;
}
return false;
}
void HashStorage::clean()
{
RS_STACK_MUTEX(mHashMtx) ;
time_t now = time(NULL) ;
time_t duration = mMaxStorageDurationDays * 24 * 3600 ; // seconds
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Cleaning hash cache." << std::endl ;
#endif
for(std::map<std::string,HashStorageInfo>::iterator it(mFiles.begin());it!=mFiles.end();)
if(it->second.time_stamp + duration < (uint64_t)now)
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << " Entry too old: " << it->first << ", ts=" << it->second.time_stamp << std::endl ;
#endif
std::map<std::string,HashStorageInfo>::iterator tmp(it) ;
++tmp ;
mFiles.erase(it) ;
it=tmp ;
mChanged = true ;
}
else
++it ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Done." << std::endl;
#endif
}
void HashStorage::locked_load()
{
unsigned char *data = NULL ;
uint32_t data_size=0;
if(!FileListIO::loadEncryptedDataFromFile(mFilePath,data,data_size))
{
std::cerr << "(EE) Cannot read hash cache." << std::endl;
return ;
}
uint32_t offset = 0 ;
HashStorageInfo info ;
uint32_t n=0;
while(offset < data_size)
if(readHashStorageInfo(data,data_size,offset,info))
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << info << std::endl;
++n ;
#endif
mFiles[info.filename] = info ;
}
free(data) ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << n << " entries loaded." << std::endl ;
#endif
}
void HashStorage::locked_save()
{
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Saving Hash Cache to file " << mFilePath << "..." << std::endl ;
#endif
unsigned char *data = NULL ;
uint32_t offset = 0 ;
uint32_t total_size = 0;
for(std::map<std::string,HashStorageInfo>::const_iterator it(mFiles.begin());it!=mFiles.end();++it)
writeHashStorageInfo(data,total_size,offset,it->second) ;
if(!FileListIO::saveEncryptedDataToFile(mFilePath,data,offset))
{
std::cerr << "(EE) Cannot save hash cache data." << std::endl;
free(data) ;
return ;
}
std::cerr << mFiles.size() << " entries saved in hash cache." << std::endl;
free(data) ;
}
bool HashStorage::readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const
{
unsigned char *section_data = NULL ;
uint32_t section_size = 0;
uint32_t section_offset = 0;
// This way, the entire section is either read or skipped. That avoids the risk of being stuck somewhere in the middle
// of a section because of some unknown field, etc.
if(!FileListIO::readField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_size))
return false;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ;
if(!FileListIO::readField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ;
free(section_data);
return true;
}
bool HashStorage::writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const
{
unsigned char *section_data = NULL ;
uint32_t section_offset = 0 ;
uint32_t section_size = 0;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_NAME ,info.filename )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SIZE ,info.size )) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_UPDATE_TS ,info.time_stamp)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_MODIF_TS ,info.modf_stamp)) return false ;
if(!FileListIO::writeField(section_data,section_size,section_offset,FILE_LIST_IO_TAG_FILE_SHA1_HASH,info.hash )) return false ;
// now write the whole string into a single section in the file
if(!FileListIO::writeField(data,total_size,offset,FILE_LIST_IO_TAG_HASH_STORAGE_ENTRY,section_data,section_offset)) return false ;
#ifdef HASHSTORAGE_DEBUG
std::cerr << "Writing hash storage section " << RsUtil::BinToHex(section_data,section_offset) << std::endl;
std::cerr << "Info.filename = " << info.filename << std::endl;
#endif
free(section_data) ;
return true;
}
std::ostream& operator<<(std::ostream& o,const HashStorage::HashStorageInfo& info)
{
return o << info.hash << " " << info.size << " " << info.filename ;
}

139
libretroshare/src/file_sharing/hash_cache.h Normal file
View File

@ -0,0 +1,139 @@
/*
* RetroShare C++ Hash cache.
*
* file_sharing/hash_cache.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <map>
#include "util/rsthreads.h"
#include "retroshare/rsfiles.h"
/*!
* \brief The HashStorageClient class
* Used by clients of the hash cache for receiving hash results when done. This is asynchrone of course since hashing
* might be quite costly.
*/
class HashStorageClient
{
public:
HashStorageClient() {}
virtual ~HashStorageClient() {}
// the result of the hashing info is sent to this method
virtual void hash_callback(uint32_t client_param, const std::string& name, const RsFileHash& hash, uint64_t size)=0;
// this method is used to check that the client param is still valid just before hashing. This avoids hashing files
// that are still in queue while removed from shared lists.
virtual bool hash_confirm(uint32_t client_param)=0 ;
};
class HashStorage: public RsTickingThread
{
public:
HashStorage(const std::string& save_file_name) ;
/*!
* \brief requestHash Requests the hash for the given file, assuming size and mod_time are the same.
*
* \param full_path Full path to reach the file
* \param size Actual file size
* \param mod_time Actual file modification time
* \param known_hash Returned hash for the file.
* \param c Hash cache client to which the hash should be sent once calculated
* \param client_param Param to be passed to the client callback. Useful if the client needs a file ID.
*
* \return true if the supplied hash info is up to date.
*/
bool requestHash(const std::string& full_path, uint64_t size, time_t mod_time, RsFileHash& known_hash, HashStorageClient *c, uint32_t client_param) ;
struct HashStorageInfo
{
std::string filename ; // full path of the file
uint64_t size ;
uint32_t time_stamp ; // last time the hash was tested/requested
uint32_t modf_stamp ;
RsFileHash hash ;
} ;
// interaction with GUI, called from p3FileLists
void setRememberHashFilesDuration(uint32_t days) { mMaxStorageDurationDays = days ; } // duration for which the hash is kept even if the file is not shared anymore
uint32_t rememberHashFilesDuration() const { return mMaxStorageDurationDays ; }
void clear() { mFiles.clear(); mChanged=true; } // drop all known hashes. Not something to do, except if you want to rehash the entire database
bool empty() const { return mFiles.empty() ; }
// Functions called by the thread
virtual void data_tick() ;
friend std::ostream& operator<<(std::ostream& o,const HashStorageInfo& info) ;
private:
/*!
* \brief clean
* This function is responsible for removing old hashes, etc
*/
void clean() ;
// loading/saving the entire hash database to a file
void locked_save() ;
void locked_load() ;
bool readHashStorageInfo(const unsigned char *data,uint32_t total_size,uint32_t& offset,HashStorageInfo& info) const;
bool writeHashStorageInfo(unsigned char *& data,uint32_t& total_size,uint32_t& offset,const HashStorageInfo& info) const;
// Local configuration and storage
uint32_t mMaxStorageDurationDays ; // maximum duration of un-requested cache entries
std::map<std::string, HashStorageInfo> mFiles ; // stored as (full_path, hash_info)
std::string mFilePath ; // file where the hash database is stored
bool mChanged ;
struct FileHashJob
{
std::string full_path;
uint64_t size ;
HashStorageClient *client;
uint32_t client_param ;
time_t ts;
};
// current work
std::map<std::string,FileHashJob> mFilesToHash ;
// thread/mutex stuff
RsMutex mHashMtx ;
bool mRunning;
uint64_t mHashCounter;
uint32_t mInactivitySleepTime ;
uint64_t mTotalSizeToHash ;
uint64_t mTotalHashedSize ;
uint64_t mTotalFilesToHash ;
time_t mLastSaveTime ;
};

1445
libretroshare/src/file_sharing/p3filelists.cc Normal file
View File

File diff suppressed because it is too large Load Diff

233
libretroshare/src/file_sharing/p3filelists.h Normal file
View File

@ -0,0 +1,233 @@
/*
* RetroShare C++ File lists service.
*
* file_sharing/p3filelists.h
*
* Copyright 2016 by Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
//
// This class is responsible for
// - maintaining a list of shared file hierarchies for each known friends
// - talking to the GUI
// - providing handles for the directory tree listing GUI
// - providing search handles for FT
// - keeping these lists up to date
// - sending our own file list to friends depending on the defined access rights
// - serving file search requests from other services such as file transfer
//
// p3FileList does the following micro-tasks:
// - tick the watchers
// - get incoming info from the service layer, which can be:
// - directory content request => the directory content is shared to the friend
// - directory content => the directory watcher is notified
// - keep two queues of update requests:
// - fast queue that is handled in highest priority. This one is used for e.g. updating while browsing.
// - slow queue that is handled slowly. Used in background update of shared directories.
//
// The file lists are not directry updated. A FileListWatcher class is responsible for this
// in every case.
//
#pragma once
#include "ft/ftsearch.h"
#include "retroshare/rsfiles.h"
#include "services/p3service.h"
#include "file_sharing/hash_cache.h"
#include "file_sharing/directory_storage.h"
#include "pqi/p3cfgmgr.h"
#include "pqi/p3linkmgr.h"
class RemoteDirectoryUpdater ;
class LocalDirectoryUpdater ;
class RemoteDirectoryStorage ;
class LocalDirectoryStorage ;
class RsFileListsSyncRequestItem ;
class RsFileListsSyncResponseItem ;
class HashStorage ;
class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService
{
public:
typedef DirectoryStorage::EntryIndex EntryIndex; // this should probably be defined elsewhere
virtual RsServiceInfo getServiceInfo();
struct RsFileListSyncRequest
{
RsPeerId peerId ;
EntryIndex index ;
// [...] more to add here
};
p3FileDatabase(p3ServiceControl *mpeers) ;
~p3FileDatabase();
/*!
* \brief forceSyncWithPeers
*
* Forces the synchronisation of the database with connected peers. This is triggered when e.g. a new group of friend is created, or when
* a friend was added/removed from a group.
*/
void forceSyncWithPeers() { NOT_IMPLEMENTED() ; }
// derived from p3Service
//
virtual int tick() ;
// ftSearch
virtual bool search(const RsFileHash &hash, FileSearchFlags hintflags, FileInfo &info) const;
virtual int SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) ;
virtual int SearchBoolExp(RsRegularExpression::Expression *exp, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
// Interface for browsing dir hierarchy
//
void stopThreads() ;
void startThreads() ;
bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) const;
// void * here is the type expected by the abstract model index from Qt. It gets turned into a DirectoryStorage::EntryIndex internally.
void requestDirUpdate(void *ref) ; // triggers an update. Used when browsing.
int RequestDirDetails(void *, DirDetails&, FileSearchFlags) const ;
uint32_t getType(void *) const ;
// proxy method used by the web UI. Dont't delete!
int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)const;
// set/update shared directories
void setSharedDirectories(const std::list<SharedDirInfo>& dirs);
void getSharedDirectories(std::list<SharedDirInfo>& dirs);
void updateShareFlags(const SharedDirInfo& info) ;
bool convertSharedFilePath(const std::string& path,std::string& fullpath);
// interface for hash caching
void setWatchPeriod(uint32_t seconds);
uint32_t watchPeriod() ;
void setWatchEnabled(bool b) ;
bool watchEnabled() ;
// interfact for directory parsing
void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed
bool inDirectoryCheck();
protected:
int filterResults(const std::list<EntryIndex>& firesults,std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const;
std::string makeRemoteFileName(const RsPeerId& pid) const;
// Derived from p3Config
//
virtual bool loadList(std::list<RsItem *>& items);
virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
virtual RsSerialiser *setupSerialiser() ;
void cleanup();
void tickRecv();
void tickSend();
private:
p3ServiceControl *mServCtrl ;
RsPeerId mOwnId ;
typedef uint64_t DirSyncRequestId ;
static DirSyncRequestId makeDirSyncReqId(const RsPeerId& peer_id, const RsFileHash &hash) ;
// utility functions to send items with some maximum size.
void splitAndSendItem(RsFileListsSyncResponseItem *ritem);
RsFileListsSyncResponseItem *recvAndRebuildItem(RsFileListsSyncResponseItem *ritem);
/*!
* \brief generateAndSendSyncRequest
* \param rds Remote directory storage for the request
* \param e Entry index to update
* \return true if the request is correctly sent.
*/
bool locked_generateAndSendSyncRequest(RemoteDirectoryStorage *rds,const DirectoryStorage::EntryIndex& e);
// File sync request queues. The fast one is used for online browsing when friends are connected.
// The slow one is used for background update of file lists.
//
std::map<DirSyncRequestId,RsFileListSyncRequest> mFastRequestQueue ;
std::map<DirSyncRequestId,RsFileListSyncRequest> mSlowRequestQueue ;
// Directory storage hierarchies
//
// The remote one is the reference for the PeerId index below:
// RemoteDirectories[ getFriendIndex(pid) - 1] = RemoteDirectoryStorage(pid)
std::vector<RemoteDirectoryStorage *> mRemoteDirectories ;
LocalDirectoryStorage *mLocalSharedDirs ;
LocalDirectoryUpdater *mLocalDirWatcher ;
// utility functions to make/get a pointer out of an (EntryIndex,PeerId) pair. This is further documented in the .cc
static bool convertEntryIndexToPointer(const EntryIndex &e, uint32_t friend_index, void *& p);
static bool convertPointerToEntryIndex(const void *p, EntryIndex& e, uint32_t& friend_index) ;
uint32_t locked_getFriendIndex(const RsPeerId& pid);
const RsPeerId& locked_getFriendFromIndex(uint32_t indx) const;
void handleDirSyncRequest (RsFileListsSyncRequestItem *) ;
void handleDirSyncResponse (RsFileListsSyncResponseItem *) ;
std::map<RsPeerId,uint32_t> mFriendIndexMap ;
std::vector<RsPeerId> mFriendIndexTab;
// Directory synchronization
//
struct DirSyncRequestData
{
RsPeerId peer_id ;
time_t request_TS ;
uint32_t flags ;
};
time_t mLastRemoteDirSweepTS ; // TS for friend list update
std::map<DirSyncRequestId,DirSyncRequestData> mPendingSyncRequests ; // pending requests, waiting for an answer
std::map<DirSyncRequestId,RsFileListsSyncResponseItem *> mPartialResponseItems;
void locked_recursSweepRemoteDirectory(RemoteDirectoryStorage *rds, DirectoryStorage::EntryIndex e, int depth);
// We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name.
//
HashStorage *mHashCache ;
// Local flags and mutexes
mutable RsMutex mFLSMtx ;
uint32_t mUpdateFlags ;
std::string mFileSharingDir ;
time_t mLastCleanupTime;
};

363
libretroshare/src/file_sharing/rsfilelistitems.cc Normal file
View File

@ -0,0 +1,363 @@
/*
* RetroShare File lists service items
*
* file_sharing/rsfilelistsitems.cc
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#include "serialiser/rsbaseserial.h"
#include "file_sharing/rsfilelistitems.h"
RsItem* RsFileListsSerialiser::deserialise(void *data, uint32_t *size)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialise()" << std::endl;
#endif
/* get the type and size */
uint32_t rstype = getRsItemId(data);
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (getRsItemService(rstype) != RS_SERVICE_TYPE_FILE_DATABASE))
return NULL; /* wrong type */
switch(getRsItemSubType(rstype))
{
case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return deserialFileListsSyncRequestItem(data, size);
case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: return deserialFileListsSyncResponseItem(data, size);
// case RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM: return deserialFileListsConfigItem (data, size);
default:
{
std::cerr << "(WW) RsFileListsSerialiser::deserialise() : unhandled item type " << getRsItemSubType(rstype) << std::endl;
return NULL;
}
}
}
uint32_t RsFileListsSerialiser::size(RsItem *item)
{
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
if(flst_item != NULL)
return flst_item->serial_size() ;
else
{
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
return 0;
}
}
bool RsFileListsSerialiser::serialise(RsItem *item, void *data, uint32_t *size)
{
RsFileListsItem *flst_item = dynamic_cast<RsFileListsItem*>(item) ;
if(flst_item != NULL)
return flst_item->serialise(data,*size) ;
else
{
std::cerr << "RsFileListsSerialiser::serialise(): Not an RsFileListsItem!" << std::endl;
return 0;
}
}
bool RsFileListsItem::serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const
{
tlvsize = serial_size() ;
offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
if(!setRsItemHeader(data, tlvsize, PacketId(), tlvsize))
{
std::cerr << "RsFileTransferItem::serialise_header(): ERROR. Not enough size!" << std::endl;
return false ;
}
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileItemSerialiser::serialiseData() Header: " << ok << std::endl;
#endif
offset += 8;
return true ;
}
bool RsFileListsSyncRequestItem::serialise(void *data, uint32_t& size) const
{
uint32_t tlvsize,offset=0;
bool ok = true;
if(!serialise_header(data,size,tlvsize,offset))
return false ;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl;
#endif
/* RsFileListsSyncMsgItem */
ok &= entry_hash.serialise(data, size, offset);
ok &= setRawUInt32(data, size, &offset, flags );
ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
ok &= setRawUInt64(data, size, &offset, request_id);
if(offset != tlvsize){
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl;
#endif
ok = false;
}
#ifdef RSSERIAL_DEBUG
if (!ok)
{
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl;
}
#endif
return ok;
}
bool RsFileListsSyncResponseItem::serialise(void *data, uint32_t& size) const
{
uint32_t tlvsize,offset=0;
bool ok = true;
if(!serialise_header(data,size,tlvsize,offset))
return false ;
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseFileListsSyncReqItem()" << std::endl;
#endif
/* RsFileListsSyncMsgItem */
ok &= entry_hash.serialise(data, size, offset);
ok &= checksum.serialise(data, size, offset);
ok &= setRawUInt32(data, size, &offset, flags );
ok &= setRawUInt32(data, size, &offset, last_known_recurs_modf_TS);
ok &= setRawUInt64(data, size, &offset, request_id);
ok &= directory_content_data.SetTlv(data,size,&offset) ;
if(offset != tlvsize){
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() FAIL Size Error! " << std::endl;
#endif
ok = false;
}
#ifdef RSSERIAL_DEBUG
if (!ok)
{
std::cerr << "RsFileListsSerialiser::serialiseNxsSynMsgItem() NOK" << std::endl;
}
#endif
return ok;
}
//============================================================================================================================//
// Deserialisation //
//============================================================================================================================//
//RsFileListsConfigItem* RsFileListsSerialiser::deserialFileListsConfigItem(void *data, uint32_t *size)
//{
// NOT_IMPLEMENTED();
//
// return NULL ;
//}
RsFileListsSyncRequestItem* RsFileListsSerialiser::deserialFileListsSyncRequestItem(void *data, uint32_t *size)
{
bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM);
uint32_t offset = 8;
RsFileListsSyncRequestItem* item = new RsFileListsSyncRequestItem();
ok &= item->entry_hash.deserialise(data, *size, offset);
ok &= getRawUInt32(data, *size, &offset, &item->flags);
ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
ok &= getRawUInt64(data, *size, &offset, &item->request_id);
if (offset != *size)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl;
#endif
/* error */
delete item;
return NULL;
}
if (!ok)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl;
#endif
delete item;
return NULL;
}
return item;
}
RsFileListsSyncResponseItem* RsFileListsSerialiser::deserialFileListsSyncResponseItem(void *data, uint32_t *size)
{
bool ok = checkItemHeader(data,size,RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM);
uint32_t offset = 8;
RsFileListsSyncResponseItem* item = new RsFileListsSyncResponseItem();
uint32_t entry_index ; // index of the directory to sync
uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
ok &= item->entry_hash.deserialise(data, *size, offset);
ok &= item->checksum.deserialise(data, *size, offset);
ok &= getRawUInt32(data, *size, &offset, &item->flags);
ok &= getRawUInt32(data, *size, &offset, &item->last_known_recurs_modf_TS);
ok &= getRawUInt64(data, *size, &offset, &item->request_id);
ok &= item->directory_content_data.GetTlv(data,*size,&offset) ;
if (offset != *size)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() FAIL size mismatch" << std::endl;
#endif
/* error */
delete item;
return NULL;
}
if (!ok)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::deserialNxsGrp() NOK" << std::endl;
#endif
delete item;
return NULL;
}
return item;
}
bool RsFileListsSerialiser::checkItemHeader(void *data,uint32_t *size,uint8_t subservice_type)
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::checkItemHeader()" << std::endl;
#endif
/* get the type and size */
uint32_t rstype = getRsItemId(data);
uint32_t rssize = getRsItemSize(data);
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_FILE_DATABASE != getRsItemService(rstype)) || (subservice_type != getRsItemSubType(rstype)))
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong type" << std::endl;
#endif
return false; /* wrong type */
}
if (*size < rssize) /* check size */
{
#ifdef RSSERIAL_DEBUG
std::cerr << "RsFileListsSerialiser::checkItemHeader() FAIL wrong size" << std::endl;
#endif
return false; /* not enough data */
}
/* set the packet length */
*size = rssize;
return true ;
}
//============================================================================================================================//
// Sizes //
//============================================================================================================================//
uint32_t RsFileListsSyncRequestItem::serial_size()const
{
uint32_t s = 8; //header size
s += RsFileHash::serial_size(); // entry hash
s += 4; // flags
s += 4; // last_known_recurs_modf_TS
s += 8; // request_id
return s;
}
uint32_t RsFileListsSyncResponseItem::serial_size()const
{
uint32_t s = 8; //header size
s += RsFileHash::serial_size(); // entry hash
s += RsFileHash::serial_size(); // checksum
s += 4; // flags
s += 4; // last_known_recurs_modf_TS
s += 8; // request_id
s += directory_content_data.TlvSize();
return s;
}
void RsFileListsSyncRequestItem::clear()
{
}
void RsFileListsSyncResponseItem::clear()
{
directory_content_data.TlvClear();
}
std::ostream& RsFileListsSyncRequestItem::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsFileListsSyncReqItem", indent);
uint16_t int_Indent = indent + 2;
printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl;
printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl;
printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl;
printRsItemEnd(out ,"RsFileListsSyncReqItem", indent);
return out;
}
std::ostream& RsFileListsSyncResponseItem::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsFileListsSyncDirItem", indent);
uint16_t int_Indent = indent + 2;
printIndent(out , int_Indent); out << "Entry hash: " << entry_hash << std::endl;
printIndent(out , int_Indent); out << "Checksum : " << checksum << std::endl;
printIndent(out , int_Indent); out << "Flags: " << (uint32_t) flags << std::endl;
printIndent(out , int_Indent); out << "Last modf TS: " << last_known_recurs_modf_TS << std::endl;
printIndent(out , int_Indent); out << "request id: " << std::hex << request_id << std::dec << std::endl;
printIndent(out , int_Indent); out << "Data size: " << directory_content_data.bin_len << std::endl;
printRsItemEnd(out ,"RsFileListsSyncDirItem", indent);
return out;
}

135
libretroshare/src/file_sharing/rsfilelistitems.h Normal file
View File

@ -0,0 +1,135 @@
/*
* RetroShare File lists service items.
*
* file_sharing/rsfilelistitems.h
*
* Copyright 2016 Mr.Alice
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com".
*
*/
#pragma once
#include <map>
#include <openssl/ssl.h>
#include "serialiser/rsserviceids.h"
#include "serialiser/rsserial.h"
#include "serialiser/rstlvbase.h"
#include "serialiser/rstlvitem.h"
#include "serialiser/rstlvkeys.h"
#include "gxs/rsgxsdata.h"
// These items have "flag type" numbers, but this is not used.
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01;
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM = 0x02;
const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03;
/*!
* Base class for filelist sync items
*/
class RsFileListsItem : public RsItem
{
public:
RsFileListsItem(uint8_t subtype)
: RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE, subtype)
{
setPriorityLevel(QOS_PRIORITY_RS_SLOW_SYNC_REQUEST); // this is the default. Someitems may be faster, on demand.
return;
}
virtual ~RsFileListsItem(){}
virtual bool serialise(void *data,uint32_t& size) const = 0 ;
virtual uint32_t serial_size() const = 0 ;
virtual void clear() = 0;
virtual std::ostream &print(std::ostream &out, uint16_t indent = 0) = 0;
bool serialise_header(void *data,uint32_t& pktsize,uint32_t& tlvsize, uint32_t& offset) const;
static const uint32_t FLAGS_SYNC_REQUEST = 0x0001 ;
static const uint32_t FLAGS_SYNC_RESPONSE = 0x0002 ;
static const uint32_t FLAGS_SYNC_DIR_CONTENT = 0x0004 ;
static const uint32_t FLAGS_ENTRY_UP_TO_DATE = 0x0008 ;
static const uint32_t FLAGS_ENTRY_WAS_REMOVED = 0x0010 ;
static const uint32_t FLAGS_SYNC_PARTIAL = 0x0020 ;
static const uint32_t FLAGS_SYNC_PARTIAL_END = 0x0040 ;
};
/*!
* Use to request synchronization on a specific directory. Also used to respond that the directory is up to date.
*/
class RsFileListsSyncRequestItem : public RsFileListsItem
{
public:
RsFileListsSyncRequestItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM) {}
virtual void clear();
virtual std::ostream &print(std::ostream &out, uint16_t indent);
virtual bool serialise(void *data,uint32_t& size) const;
virtual uint32_t serial_size() const ;
RsFileHash entry_hash ; // hash of the directory to sync
uint32_t flags; // used to say that it's a request or a response, say that the directory has been removed, ask for further update, etc.
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
};
class RsFileListsSyncResponseItem : public RsFileListsItem
{
public:
RsFileListsSyncResponseItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM) {}
virtual void clear();
virtual std::ostream &print(std::ostream &out, uint16_t indent);
virtual bool serialise(void *data,uint32_t& size) const;
virtual uint32_t serial_size() const ;
RsFileHash entry_hash ; // hash of the directory to sync
RsFileHash checksum ; // checksum of the bindary data, for checking
uint32_t flags; // is it a partial/final item (used for large items only)
uint32_t last_known_recurs_modf_TS; // time of last modification, computed over all files+directories below.
uint64_t request_id; // use to determine if changes that have occured since last hash
RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser.
};
class RsFileListsSerialiser : public RsSerialType
{
public:
RsFileListsSerialiser() : RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_FILE_DATABASE) {}
virtual ~RsFileListsSerialiser() {}
virtual uint32_t size(RsItem *item);
virtual bool serialise(RsItem *item, void *data, uint32_t *size);
virtual RsItem* deserialise(void *data, uint32_t *size);
private:
RsFileListsSyncRequestItem *deserialFileListsSyncRequestItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
RsFileListsSyncResponseItem *deserialFileListsSyncResponseItem(void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
// RsFileListsSyncResponseItem *deserialFileListsConfigItem (void *data, uint32_t *size); /* RS_PKT_SUBTYPE_SYNC_GRP */
bool checkItemHeader(void *data, uint32_t *size, uint8_t subservice_type);
};

172
libretroshare/src/ft/ftcontroller.cc
View File

@ -73,10 +73,8 @@
static const int32_t SAVE_TRANSFERS_DELAY = 301 ; // save transfer progress every 301 seconds.
static const int32_t INACTIVE_CHUNKS_CHECK_DELAY = 240 ; // time after which an inactive chunk is released
static const int32_t MAX_TIME_INACTIVE_REQUEUED = 120 ; // time after which an inactive ftFileControl is bt-queued
static const int32_t TIMOUT_CACHE_FILE_TRANSFER = 800 ; // time after which cache transfer gets cancelled if inactive.
static const int32_t FT_FILECONTROL_QUEUE_ADD_END = 0 ;
static const int32_t FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE = 1 ;
const uint32_t FT_CNTRL_STANDARD_RATE = 10 * 1024 * 1024;
const uint32_t FT_CNTRL_SLOW_RATE = 100 * 1024;
@ -99,8 +97,8 @@ ftFileControl::ftFileControl(std::string fname,
return;
}
ftController::ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId)
:CacheTransfer(cs), p3Config(),
ftController::ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId)
: p3Config(),
last_save_time(0),
last_clean_time(0),
mDataplex(dm),
@ -231,7 +229,6 @@ void ftController::data_tick()
time_t now = time(NULL) ;
if(now > last_save_time + SAVE_TRANSFERS_DELAY)
{
cleanCacheDownloads() ;
searchForDirectSources() ;
IndicateConfigChanged() ;
@ -282,7 +279,6 @@ void ftController::searchForDirectSources()
for(std::map<RsFileHash,ftFileControl*>::iterator it(mDownloads.begin()); it != mDownloads.end(); ++it)
if(it->second->mState != ftFileControl::QUEUED && it->second->mState != ftFileControl::PAUSED)
if(! (it->second->mFlags & RS_FILE_REQ_CACHE))
{
FileInfo info ; // info needs to be re-allocated each time, to start with a clear list of peers (it's not cleared down there)
@ -335,37 +331,6 @@ void ftController::setPriority(const RsFileHash& hash,DwlSpeed p)
it->second->mTransfer->setDownloadPriority(p) ;
}
void ftController::cleanCacheDownloads()
{
std::vector<RsFileHash> toCancel ;
time_t now = time(NULL) ;
{
RsStackMutex stack(ctrlMutex); /******* LOCKED ********/
for(std::map<RsFileHash,ftFileControl*>::iterator it(mDownloads.begin());it!=mDownloads.end();++it)
if (((it->second)->mFlags & RS_FILE_REQ_CACHE) && it->second->mState != ftFileControl::DOWNLOADING)
// check if a cache file is downloaded, if the case, timeout the transfer after TIMOUT_CACHE_FILE_TRANSFER
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::run() cache transfer found. age of this tranfer is :" << (int)(time(NULL) - (it->second)->mCreateTime);
std::cerr << std::endl;
#endif
if (now > (it->second)->mCreator->creationTimeStamp() + TIMOUT_CACHE_FILE_TRANSFER)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::run() cache transfer to old. Cancelling transfer. Hash :" << (it->second)->mHash << ", time=" << (it->second)->mCreateTime << ", now = " << time(NULL) ;
std::cerr << std::endl;
#endif
toCancel.push_back((it->second)->mHash);
}
}
}
for(uint32_t i=0;i<toCancel.size();++i)
FileCancel(toCancel[i]);
}
/* Called every 10 seconds or so */
void ftController::checkDownloadQueue()
{
@ -446,16 +411,12 @@ void ftController::checkDownloadQueue()
for(uint32_t p=0;p<_queue.size();++p)
{
if(p < _min_prioritized_transfers)
if(_queue[p]->mFlags & RS_FILE_REQ_CACHE) // cache file. add to potential move list
to_move_before.push_back(p) ;
else
++user_transfers ; // count one more user file in the prioritized range.
else
{
if(to_move_after.size() + user_transfers >= _min_prioritized_transfers) // we caught enough transfers to move back to the top of the queue.
break ;
if(!(_queue[p]->mFlags & RS_FILE_REQ_CACHE)) // non cache file. add to potential move list
to_move_after.push_back(p) ;
}
}
@ -482,27 +443,6 @@ void ftController::locked_addToQueue(ftFileControl* ftfc,int add_strategy)
case FT_FILECONTROL_QUEUE_ADD_END: _queue.push_back(ftfc) ;
locked_checkQueueElement(_queue.size()-1) ;
break ;
case FT_FILECONTROL_QUEUE_ADD_AFTER_CACHE:
{
// We add the transfer just before the first non cache transfer.
// This is costly, so only use this in case we really need it.
//
uint32_t pos =0;
while(pos < _queue.size() && (pos < _min_prioritized_transfers || (_queue[pos]->mFlags & RS_FILE_REQ_CACHE)>0) )
++pos ;
_queue.push_back(NULL) ;
for(int i=int(_queue.size())-1;i>(int)pos;--i)
{
_queue[i] = _queue[i-1] ;
locked_checkQueueElement(i) ;
}
_queue[pos] = ftfc ;
locked_checkQueueElement(pos) ;
}
break ;
}
}
@ -889,12 +829,8 @@ bool ftController::completeFile(const RsFileHash& hash)
locked_queueRemove(it->second->mQueuePosition) ;
/* switch map */
if (!(fc->mFlags & RS_FILE_REQ_CACHE)) /* clean up completed cache files automatically */
{
mCompleted[fc->mHash] = fc;
completeCount = mCompleted.size();
} else
delete fc ;
mDownloads.erase(it);
@ -910,35 +846,7 @@ bool ftController::completeFile(const RsFileHash& hash)
/* If it has a callback - do it now */
if(flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA))// | RS_FILE_HINTS_MEDIA))
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() doing Callback, callbackflags:" << (flags & ( RS_FILE_REQ_CACHE | RS_FILE_REQ_EXTRA ));//| RS_FILE_HINTS_MEDIA)) ;
std::cerr << std::endl;
#endif
if(flags & RS_FILE_REQ_CACHE)
{
/* callback */
if (state == ftFileControl::COMPLETED)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() doing Callback : Success";
std::cerr << std::endl;
#endif
CompletedCache(hash);
}
else
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() Cache Callback : Failed";
std::cerr << std::endl;
#endif
FailedCache(hash);
}
}
if(flags & RS_FILE_REQ_EXTRA)
if(flags & RS_FILE_REQ_EXTRA)// | RS_FILE_HINTS_MEDIA))
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::completeFile() adding to ExtraList";
@ -947,15 +855,6 @@ bool ftController::completeFile(const RsFileHash& hash)
mExtraList->addExtraFile(path, hash, size, period, extraflags);
}
// if(flags & RS_FILE_HINTS_MEDIA)
// {
//#ifdef CONTROL_DEBUG
// std::cerr << "ftController::completeFile() NULL MEDIA callback";
// std::cerr << std::endl;
//#endif
// }
}
else
{
#ifdef CONTROL_DEBUG
@ -965,14 +864,12 @@ bool ftController::completeFile(const RsFileHash& hash)
}
/* Notify GUI */
if ((flags & RS_FILE_REQ_CACHE) == 0) {
RsServer::notify()->AddPopupMessage(RS_POPUP_DOWNLOAD, hash.toStdString(), name, "");
RsServer::notify()->notifyDownloadComplete(hash.toStdString());
RsServer::notify()->notifyDownloadCompleteCount(completeCount);
rsFiles->ForceDirectoryCheck() ;
}
IndicateConfigChanged(); /* completed transfer -> save */
return true;
@ -1134,7 +1031,6 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash
// remove the sources from the list, if they don't have clearance for direct transfer. This happens only for non cache files.
//
if(!(flags & RS_FILE_REQ_CACHE))
for(std::list<RsPeerId>::iterator it = srcIds.begin(); it != srcIds.end(); )
if(!(rsPeers->servicePermissionFlags(*it) & RS_NODE_PERM_DIRECT_DL))
{
@ -1280,7 +1176,7 @@ bool ftController::FileRequest(const std::string& fname, const RsFileHash& hash
if(flags & RS_FILE_REQ_ANONYMOUS_ROUTING)
mTurtle->monitorTunnels(hash,mFtServer,true) ;
bool assume_availability = flags & RS_FILE_REQ_CACHE ; // assume availability for cache files
bool assume_availability = false;
ftFileCreator *fc = new ftFileCreator(savepath, size, hash,assume_availability);
ftTransferModule *tm = new ftTransferModule(fc, mDataplex,this);
@ -1911,58 +1807,6 @@ void ftController::statusChange(const std::list<pqiServicePeer> &plist)
}
}
/* Cache Interface */
bool ftController::RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftController::RequestCacheFile(" << id << ",";
std::cerr << path << "," << hash << "," << size << ")";
std::cerr << std::endl;
#endif
/* Request File */
std::list<RsPeerId> ids;
ids.push_back(id);
FileInfo info ;
if(mSearch->search(hash, RS_FILE_HINTS_CACHE, info))
{
#ifdef CONTROL_DEBUG
std::cerr << "I already have this file:" << std::endl ;
std::cerr << " path: " << info.path << std::endl ;
std::cerr << " fname: " << info.fname << std::endl ;
std::cerr << " hash: " << info.hash << std::endl ;
std::cerr << "Copying it !!" << std::endl ;
#endif
if(info.size > 0 && copyFile(info.path,path+"/"+hash.toStdString()))
{
CompletedCache(hash);
return true ;
}
else
return false ;
}
FileRequest(hash.toStdString(), hash, size, path, RS_FILE_REQ_CACHE | RS_FILE_REQ_NO_SEARCH, ids);
return true;
}
bool ftController::CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size)
{
std::cerr << "ftController::CancelCacheFile(" << id << ",";
std::cerr << path << "," << hash << "," << size << ")";
std::cerr << std::endl;
#ifdef CONTROL_DEBUG
#endif
return FileCancel(hash);
}
const std::string active_downloads_size_ss("MAX_ACTIVE_DOWNLOADS");
const std::string min_prioritized_downl_ss("MIN_PRORITIZED_DOWNLOADS");
const std::string download_dir_ss("DOWN_DIR");
@ -2056,14 +1900,6 @@ bool ftController::saveList(bool &cleanup, std::list<RsItem *>& saveData)
/* ignore cache files. As this is small files, better download them again from scratch at restart.*/
if (fit->second->mFlags & RS_FILE_REQ_CACHE)
{
#ifdef CONTROL_DEBUG
std::cerr << "ftcontroller::saveList(): Not saving (callback) file entry " << fit->second->mName << ", " << fit->second->mHash << ", " << fit->second->mSize << std::endl ;
#endif
continue;
}
// Node: We still save finished transfers. This keeps transfers that are
// in checking mode. Finished or checked transfers will restart and
// immediately terminate/recheck at next startup.

9
libretroshare/src/ft/ftcontroller.h
View File

@ -48,7 +48,6 @@ class ftDataMultiplex;
class p3turtle ;
class p3ServiceControl;
#include "dbase/cachestrapper.h"
#include "util/rsthreads.h"
#include "pqi/pqiservicemonitor.h"
#include "pqi/p3cfgmgr.h"
@ -113,12 +112,12 @@ class ftPendingRequest
};
class ftController: public CacheTransfer, public RsTickingThread, public pqiServiceMonitor, public p3Config
class ftController: public RsTickingThread, public pqiServiceMonitor, public p3Config
{
public:
/* Setup */
ftController(CacheStrapper *cs, ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId);
ftController(ftDataMultiplex *dm, p3ServiceControl *sc, uint32_t ftServiceId);
void setFtSearchNExtra(ftSearch *, ftExtraList *);
void setTurtleRouter(p3turtle *) ;
@ -195,10 +194,6 @@ class ftController: public CacheTransfer, public RsTickingThread, public pqiServ
protected:
virtual bool RequestCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
virtual bool CancelCacheFile(const RsPeerId& id, std::string path, const RsFileHash& hash, uint64_t size);
void cleanCacheDownloads() ;
void searchForDirectSources() ;
void tickTransfers() ;

2
libretroshare/src/ft/ftdatamultiplex.cc
View File

@ -1104,8 +1104,6 @@ bool ftDataMultiplex::handleSearchRequest(const RsPeerId& peerId, const RsFileHa
if(rsTurtle->isTurtlePeer(peerId))
hintflags |= RS_FILE_HINTS_NETWORK_WIDE ;
else
hintflags |= RS_FILE_HINTS_CACHE ;
if(mSearch->search(hash, hintflags, info))
{

385
libretroshare/src/ft/ftserver.cc
View File

@ -31,6 +31,7 @@
#include "retroshare/rspeers.h"
const int ftserverzone = 29539;
#include "file_sharing/p3filelists.h"
#include "ft/ftturtlefiletransferitem.h"
#include "ft/ftserver.h"
#include "ft/ftextralist.h"
@ -43,11 +44,6 @@ const int ftserverzone = 29539;
#include "pqi/p3notify.h"
#include "rsserver/p3face.h"
// Includes CacheStrapper / FiMonitor / FiStore for us.
#include "ft/ftdbase.h"
#include "pqi/pqi.h"
#include "pqi/p3linkmgr.h"
@ -64,15 +60,11 @@ static const time_t FILE_TRANSFER_LOW_PRIORITY_TASKS_PERIOD = 5 ; // low priorit
/* Setup */
ftServer::ftServer(p3PeerMgr *pm, p3ServiceControl *sc)
: p3Service(),
mPeerMgr(pm),
mServiceCtrl(sc),
mCacheStrapper(NULL),
mFiStore(NULL), mFiMon(NULL),
mPeerMgr(pm), mServiceCtrl(sc),
mFileDatabase(NULL),
mFtController(NULL), mFtExtra(NULL),
mFtDataplex(NULL), mFtSearch(NULL), srvMutex("ftServer")
{
mCacheStrapper = new ftCacheStrapper(sc, getServiceInfo().mServiceType);
addSerialType(new RsFileTransferSerialiser()) ;
}
@ -145,33 +137,25 @@ void ftServer::SetupFtServer()
mFtDataplex = new ftDataMultiplex(ownId, this, mFtSearch);
/* make Controller */
mFtController = new ftController(mCacheStrapper, mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType);
mFtController = new ftController(mFtDataplex, mServiceCtrl, getServiceInfo().mServiceType);
mFtController -> setFtSearchNExtra(mFtSearch, mFtExtra);
std::string tmppath = ".";
mFtController->setPartialsDirectory(tmppath);
mFtController->setDownloadDirectory(tmppath);
/* Make Cache Source/Store */
mFiStore = new ftFiStore(mCacheStrapper, mFtController, mPeerMgr, ownId, remotecachedir);
mFiMon = new ftFiMonitor(mCacheStrapper,localcachedir, ownId,mConfigPath);
/* now add the set to the cachestrapper */
CachePair cp(mFiMon, mFiStore, CacheId(RS_SERVICE_TYPE_FILE_INDEX, 0));
mCacheStrapper -> addCachePair(cp);
/* complete search setup */
mFtSearch->addSearchMode(mCacheStrapper, RS_FILE_HINTS_CACHE);
mFtSearch->addSearchMode(mFtExtra, RS_FILE_HINTS_EXTRA);
mFtSearch->addSearchMode(mFiMon, RS_FILE_HINTS_LOCAL);
mFtSearch->addSearchMode(mFiStore, RS_FILE_HINTS_REMOTE);
mServiceCtrl->registerServiceMonitor(mFtController, getServiceInfo().mServiceType);
mServiceCtrl->registerServiceMonitor(mCacheStrapper, getServiceInfo().mServiceType);
return;
}
void ftServer::connectToFileDatabase(p3FileDatabase *fdb)
{
mFileDatabase = fdb ;
mFtSearch->addSearchMode(fdb, RS_FILE_HINTS_LOCAL | RS_FILE_HINTS_REMOTE);
}
void ftServer::connectToTurtleRouter(p3turtle *fts)
{
mTurtleRouter = fts ;
@ -193,7 +177,7 @@ void ftServer::StartupThreads()
/* startup Monitor Thread */
/* startup the FileMonitor (after cache load) */
/* start it up */
mFiMon->start("ft monitor");
mFileDatabase->startThreads();
/* Controller thread */
mFtController->start("ft ctrl");
@ -210,9 +194,6 @@ void ftServer::StopThreads()
/* stop Controller thread */
mFtController->join();
/* stop Monitor Thread */
mFiMon->join();
/* self contained threads */
/* stop ExtraList Thread */
mFtExtra->join();
@ -223,24 +204,15 @@ void ftServer::StopThreads()
delete (mFtController);
mFtController = NULL;
delete (mFiMon);
mFiMon = NULL;
delete (mFtExtra);
mFtExtra = NULL;
}
CacheStrapper *ftServer::getCacheStrapper()
{
return mCacheStrapper;
/* stop Monitor Thread */
mFileDatabase->stopThreads();
delete mFileDatabase;
mFileDatabase = NULL ;
}
CacheTransfer *ftServer::getCacheTransfer()
{
return mFtController;
}
/***************************************************************/
/********************** RsFiles Interface **********************/
/***************************************************************/
@ -256,11 +228,6 @@ bool ftServer::ResumeTransfers()
return true;
}
bool ftServer::checkHash(const RsFileHash& /*hash*/, std::string& /*error_string*/)
{
return true ;
}
bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& requested_size,uint8_t *data)
{
return mFtDataplex->getFileData(hash, offset, requested_size,data);
@ -268,19 +235,11 @@ bool ftServer::getFileData(const RsFileHash& hash, uint64_t offset, uint32_t& re
bool ftServer::alreadyHaveFile(const RsFileHash& hash, FileInfo &info)
{
return mFtController->alreadyHaveFile(hash, info);
return mFileDatabase->search(hash, RS_FILE_HINTS_LOCAL, info);
}
bool ftServer::FileRequest(const std::string& fname, const RsFileHash& hash, uint64_t size, const std::string& dest, TransferRequestFlags flags, const std::list<RsPeerId>& srcIds)
{
std::string error_string ;
if(!checkHash(hash,error_string))
{
RsServer::notify()->notifyErrorMsg(0,0,"Error handling hash \""+hash.toStdString()+"\". This hash appears to be invalid(Error string=\""+error_string+"\"). This is probably due an bad handling of strings.") ;
return false ;
}
std::cerr << "Requesting " << fname << std::endl ;
if(!mFtController->FileRequest(fname, hash, size, dest, flags, srcIds))
@ -335,14 +294,6 @@ bool ftServer::FileClearCompleted()
{
return mFtController->FileClearCompleted();
}
void ftServer::setMinPrioritizedTransfers(uint32_t s)
{
mFtController->setMinPrioritizedTransfers(s) ;
}
uint32_t ftServer::getMinPrioritizedTransfers()
{
return mFtController->getMinPrioritizedTransfers() ;
}
void ftServer::setQueueSize(uint32_t s)
{
mFtController->setQueueSize(s) ;
@ -381,6 +332,11 @@ bool ftServer::FileDownloadChunksDetails(const RsFileHash& hash,FileChunksInfo&
return mFtController->getFileDownloadChunksDetails(hash,info);
}
void ftServer::requestDirUpdate(void *ref)
{
mFileDatabase->requestDirUpdate(ref) ;
}
/* Directory Handling */
void ftServer::setDownloadDirectory(std::string path)
{
@ -402,16 +358,18 @@ std::string ftServer::getPartialsDirectory()
return mFtController->getPartialsDirectory();
}
/***************************************************************/
/************************* Other Access ************************/
/***************************************************************/
bool ftServer::copyFile(const std::string& source, const std::string& dest)
{
return mFtController->copyFile(source, dest);
}
void ftServer::FileDownloads(std::list<RsFileHash> &hashs)
{
mFtController->FileDownloads(hashs);
/* this only contains downloads.... not completed */
//return mFtDataplex->FileDownloads(hashs);
}
bool ftServer::FileUploadChunksDetails(const RsFileHash& hash,const RsPeerId& peer_id,CompressedChunkMap& cmap)
@ -438,7 +396,7 @@ bool ftServer::FileDetails(const RsFileHash &hash, FileSearchFlags hintflags, Fi
// file, we skip the call to fileDetails() for efficiency reasons.
//
FileInfo info2 ;
if( (!(info.transfer_info_flags & RS_FILE_REQ_CACHE)) && mFtController->FileDetails(hash, info2))
if(mFtController->FileDetails(hash, info2))
info.fname = info2.fname ;
return true ;
@ -543,8 +501,7 @@ bool ftServer::handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_i
/******************* ExtraFileList Access **********************/
/***************************************************************/
bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size,
uint32_t period, TransferRequestFlags flags)
bool ftServer::ExtraFileAdd(std::string fname, const RsFileHash& hash, uint64_t size, uint32_t period, TransferRequestFlags flags)
{
return mFtExtra->addExtraFile(fname, hash, size, period, flags);
}
@ -564,122 +521,53 @@ bool ftServer::ExtraFileStatus(std::string localpath, FileInfo &info)
return mFtExtra->hashExtraFileDone(localpath, info);
}
bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size,
std::string destpath)
bool ftServer::ExtraFileMove(std::string fname, const RsFileHash& hash, uint64_t size, std::string destpath)
{
return mFtExtra->moveExtraFile(fname, hash, size, destpath);
}
/***************************************************************/
/******************** Directory Listing ************************/
/***************************************************************/
int ftServer::RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::RequestDirDetails(uid:" << uid;
std::cerr << ", path:" << path << ", ...) -> mFiStore";
std::cerr << std::endl;
return mFileDatabase->RequestDirDetails(uid, path, details);
}
if (!mFiStore)
bool ftServer::findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
return mFileDatabase->findChildPointer(ref,row,result,flags) ;
}
#endif
if(uid == mServiceCtrl->getOwnId())
return mFiMon->RequestDirDetails(path, details);
else
return mFiStore->RequestDirDetails(uid, path, details);
}
int ftServer::RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::RequestDirDetails(ref:" << ref;
std::cerr << ", flags:" << flags << ", ...) -> mFiStore";
std::cerr << std::endl;
if (!mFiStore)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
}
#endif
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->RequestDirDetails(ref, details, flags);
else
return mFiStore->RequestDirDetails(ref, details, flags);
return mFileDatabase->RequestDirDetails(ref,details,flags) ;
}
uint32_t ftServer::getType(void *ref, FileSearchFlags flags)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::RequestDirDetails(ref:" << ref;
std::cerr << ", flags:" << flags << ", ...) -> mFiStore";
std::cerr << std::endl;
if (!mFiStore)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
}
#endif
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->getType(ref);
else
return mFiStore->getType(ref);
return mFileDatabase->getType(ref) ;
}
/***************************************************************/
/******************** Search Interface *************************/
/***************************************************************/
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags)
{
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchKeywords(keywords, results,flags,RsPeerId());
else
return mFiStore->SearchKeywords(keywords, results,flags);
return 0 ;
return mFileDatabase->SearchKeywords(keywords, results,flags,RsPeerId());
}
int ftServer::SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::SearchKeywords()";
std::cerr << std::endl;
return mFileDatabase->SearchKeywords(keywords, results,flags,peer_id);
}
if (!mFiStore)
int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags)
{
std::cerr << "mFiStore not SET yet = FAIL";
std::cerr << std::endl;
return mFileDatabase->SearchBoolExp(exp, results,flags,RsPeerId());
}
#endif
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchKeywords(keywords, results,flags,peer_id);
else
return mFiStore->SearchKeywords(keywords, results,flags);
}
int ftServer::SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags)
int ftServer::SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
{
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchBoolExp(exp,results,flags,RsPeerId()) ;
else
return mFiStore->searchBoolExp(exp, results);
return 0 ;
return mFileDatabase->SearchBoolExp(exp,results,flags,peer_id) ;
}
int ftServer::SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id)
{
if(flags & RS_FILE_HINTS_LOCAL)
return mFiMon->SearchBoolExp(exp,results,flags,peer_id) ;
else
return mFiStore->searchBoolExp(exp, results);
}
/***************************************************************/
/*************** Local Shared Dir Interface ********************/
@ -687,38 +575,33 @@ int ftServer::SearchBoolExp(Expression * exp, std::list<DirDetails> &results,Fil
bool ftServer::ConvertSharedFilePath(std::string path, std::string &fullpath)
{
return mFiMon->convertSharedFilePath(path, fullpath);
return mFileDatabase->convertSharedFilePath(path, fullpath);
}
void ftServer::updateSinceGroupPermissionsChanged()
{
mFiMon->forceDirListsRebuildAndSend();
mFileDatabase->forceSyncWithPeers();
}
void ftServer::ForceDirectoryCheck()
{
mFiMon->forceDirectoryCheck();
mFileDatabase->forceDirectoryCheck();
return;
}
bool ftServer::InDirectoryCheck()
{
return mFiMon->inDirectoryCheck();
}
bool ftServer::copyFile(const std::string& source, const std::string& dest)
{
return mFtController->copyFile(source, dest);
return mFileDatabase->inDirectoryCheck();
}
bool ftServer::getSharedDirectories(std::list<SharedDirInfo> &dirs)
{
mFiMon->getSharedDirectories(dirs);
mFileDatabase->getSharedDirectories(dirs);
return true;
}
bool ftServer::setSharedDirectories(std::list<SharedDirInfo> &dirs)
{
mFiMon->setSharedDirectories(dirs);
mFileDatabase->setSharedDirectories(dirs);
return true;
}
@ -728,7 +611,7 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir)
_dir.filename = RsDirUtil::convertPathToUnix(_dir.filename);
std::list<SharedDirInfo> dirList;
mFiMon->getSharedDirectories(dirList);
mFileDatabase->getSharedDirectories(dirList);
// check that the directory is not already in the list.
for(std::list<SharedDirInfo>::const_iterator it(dirList.begin());it!=dirList.end();++it)
@ -738,13 +621,13 @@ bool ftServer::addSharedDirectory(const SharedDirInfo& dir)
// ok then, add the shared directory.
dirList.push_back(_dir);
mFiMon->setSharedDirectories(dirList);
mFileDatabase->setSharedDirectories(dirList);
return true;
}
bool ftServer::updateShareFlags(const SharedDirInfo& info)
{
mFiMon->updateShareFlags(info);
mFileDatabase->updateShareFlags(info);
return true ;
}
@ -761,7 +644,7 @@ bool ftServer::removeSharedDirectory(std::string dir)
std::cerr << std::endl;
#endif
mFiMon->getSharedDirectories(dirList);
mFileDatabase->getSharedDirectories(dirList);
#ifdef SERVER_DEBUG
for(it = dirList.begin(); it != dirList.end(); ++it)
@ -776,12 +659,7 @@ bool ftServer::removeSharedDirectory(std::string dir)
if(it == dirList.end())
{
#ifdef SERVER_DEBUG
std::cerr << "ftServer::removeSharedDirectory()";
std::cerr << " Cannot Find Directory... Fail";
std::cerr << std::endl;
#endif
std::cerr << "(EE) ftServer::removeSharedDirectory(): Cannot Find Directory... Fail" << std::endl;
return false;
}
@ -793,44 +671,20 @@ bool ftServer::removeSharedDirectory(std::string dir)
#endif
dirList.erase(it);
mFiMon->setSharedDirectories(dirList);
mFileDatabase->setSharedDirectories(dirList);
return true;
}
void ftServer::setWatchPeriod(int minutes)
{
mFiMon->setWatchPeriod(minutes*60) ;
}
int ftServer::watchPeriod() const
{
return mFiMon->watchPeriod()/60 ;
}
bool ftServer::watchEnabled() { return mFileDatabase->watchEnabled() ; }
int ftServer::watchPeriod() const { return mFileDatabase->watchPeriod()/60 ; }
void ftServer::setRememberHashFiles(bool b)
{
mFiMon->setRememberHashCache(b) ;
}
bool ftServer::rememberHashFiles() const
{
return mFiMon->rememberHashCache() ;
}
void ftServer::setRememberHashFilesDuration(uint32_t days)
{
mFiMon->setRememberHashCacheDuration(days) ;
}
uint32_t ftServer::rememberHashFilesDuration() const
{
return mFiMon->rememberHashCacheDuration() ;
}
void ftServer::clearHashCache()
{
mFiMon->clearHashCache() ;
}
void ftServer::setWatchEnabled(bool b) { mFileDatabase->setWatchEnabled(b) ; }
void ftServer::setWatchPeriod(int minutes) { mFileDatabase->setWatchPeriod(minutes*60) ; }
bool ftServer::getShareDownloadDirectory()
{
std::list<SharedDirInfo> dirList;
mFiMon->getSharedDirectories(dirList);
mFileDatabase->getSharedDirectories(dirList);
std::string dir = mFtController->getDownloadDirectory();
@ -844,7 +698,8 @@ bool ftServer::getShareDownloadDirectory()
bool ftServer::shareDownloadDirectory(bool share)
{
if (share) {
if (share)
{
/* Share */
SharedDirInfo inf ;
inf.filename = mFtController->getDownloadDirectory();
@ -852,48 +707,22 @@ bool ftServer::shareDownloadDirectory(bool share)
return addSharedDirectory(inf);
}
else
{
/* Unshare */
std::string dir = mFtController->getDownloadDirectory();
return removeSharedDirectory(dir);
}
}
/***************************************************************/
/****************** End of RsFiles Interface *******************/
/***************************************************************/
/***************************************************************/
/**************** Config Interface *****************************/
/***************************************************************/
/* Key Functions to be overloaded for Full Configuration */
RsSerialiser *ftServer::setupSerialiser()
{
RsSerialiser *rss = new RsSerialiser ;
rss->addSerialType(new RsFileTransferSerialiser) ;
//rss->addSerialType(new RsGeneralConfigSerialiser());
return rss ;
}
bool ftServer::saveList(bool &/*cleanup*/, std::list<RsItem *>& /*list*/)
{
return true;
}
bool ftServer::loadList(std::list<RsItem *>& /*load*/)
{
return true;
}
bool ftServer::loadConfigMap(std::map<std::string, std::string> &/*configMap*/)
{
return true;
}
//bool ftServer::loadConfigMap(std::map<std::string, std::string> &/*configMap*/)
//{
// return true;
//}
/***************************************************************/
/********************** Data Flow **********************/
@ -1257,9 +1086,6 @@ int ftServer::tick()
if(handleIncoming())
moreToTick = true;
if(handleCacheData())
moreToTick = true;
static time_t last_law_priority_tasks_handling_time = 0 ;
time_t now = time(NULL) ;
@ -1275,45 +1101,6 @@ int ftServer::tick()
return moreToTick;
}
bool ftServer::handleCacheData()
{
std::list<std::pair<RsPeerId, RsCacheData> > cacheUpdates;
std::list<std::pair<RsPeerId, RsCacheData> >::iterator it;
int i=0 ;
#ifdef SERVER_DEBUG_CACHE
std::cerr << "handleCacheData() " << std::endl;
#endif
mCacheStrapper->getCacheUpdates(cacheUpdates);
for(it = cacheUpdates.begin(); it != cacheUpdates.end(); ++it)
{
/* construct reply */
RsFileTransferCacheItem *ci = new RsFileTransferCacheItem();
/* id from incoming */
ci -> PeerId(it->first);
ci -> file.hash = (it->second).hash;
ci -> file.name = (it->second).name;
ci -> file.path = ""; // (it->second).path;
ci -> file.filesize = (it->second).size;
ci -> cacheType = (it->second).cid.type;
ci -> cacheSubId = (it->second).cid.subid;
#ifdef SERVER_DEBUG_CACHE
std::string out2 = "Outgoing CacheStrapper Update -> RsCacheItem:\n";
ci -> print_string(out2);
std::cerr << out2 << std::endl;
#endif
sendItem(ci);
i++;
}
return i>0 ;
}
int ftServer::handleIncoming()
{
// now File Input.
@ -1411,42 +1198,6 @@ int ftServer::handleIncoming()
}
}
break ;
case RS_PKT_SUBTYPE_FT_CACHE_ITEM:
{
RsFileTransferCacheItem *ci = dynamic_cast<RsFileTransferCacheItem*>(item) ;
if (ci)
{
#ifdef SERVER_DEBUG_CACHE
std::cerr << "ftServer::handleIncoming: received cache item hash=" << ci->file.hash << ". from peer " << ci->PeerId() << std::endl;
#endif
/* these go to the CacheStrapper! */
RsCacheData data;
data.pid = ci->PeerId();
data.cid = CacheId(ci->cacheType, ci->cacheSubId);
data.path = ci->file.path;
data.name = ci->file.name;
data.hash = ci->file.hash;
data.size = ci->file.filesize;
data.recvd = time(NULL) ;
mCacheStrapper->recvCacheResponse(data, time(NULL));
}
}
break ;
// case RS_PKT_SUBTYPE_FT_CACHE_REQUEST:
// {
// // do nothing
// RsFileTransferCacheRequestItem *cr = dynamic_cast<RsFileTransferCacheRequestItem*>(item) ;
// if (cr)
// {
//#ifdef SERVER_DEBUG_CACHE
// std::cerr << "ftServer::handleIncoming: received cache request from peer " << cr->PeerId() << std::endl;
//#endif
// }
// }
// break ;
}
delete item ;
@ -1465,7 +1216,7 @@ int ftServer::handleIncoming()
bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
{
/* add all the subbits to config mgr */
cfgmgr->addConfiguration("ft_shared.cfg", mFiMon);
cfgmgr->addConfiguration("ft_database.cfg", mFileDatabase);
cfgmgr->addConfiguration("ft_extra.cfg", mFtExtra);
cfgmgr->addConfiguration("ft_transfers.cfg", mFtController);

53
libretroshare/src/ft/ftserver.h
View File

@ -47,17 +47,13 @@
#include "turtle/turtleclientservice.h"
#include "services/p3service.h"
#include "retroshare/rsfiles.h"
//#include "dbase/cachestrapper.h"
#include "pqi/pqi.h"
#include "pqi/p3cfgmgr.h"
class p3ConnectMgr;
class p3FileDatabase;
class CacheStrapper;
class CacheTransfer;
class ftCacheStrapper;
class ftFiStore;
class ftFiMonitor;
@ -70,6 +66,7 @@ class p3turtle;
class p3PeerMgr;
class p3ServiceControl;
class p3FileDatabase;
class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTurtleClientService
{
@ -89,18 +86,12 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
/* add Config Items (Extra, Controller) */
void addConfigComponents(p3ConfigMgr *mgr);
virtual CacheStrapper *getCacheStrapper();
virtual CacheTransfer *getCacheTransfer();
const RsPeerId& OwnId();
/* Final Setup (once everything is assigned) */
void SetupFtServer() ;
virtual void connectToTurtleRouter(p3turtle *p) ;
// Checks that the given hash is well formed. Used to chase
// string bugs.
static bool checkHash(const RsFileHash& hash,std::string& error_string) ;
virtual void connectToFileDatabase(p3FileDatabase *b);
// Implements RsTurtleClientService
//
@ -149,8 +140,6 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
/***
* Control of Downloads Priority.
***/
virtual uint32_t getMinPrioritizedTransfers() ;
virtual void setMinPrioritizedTransfers(uint32_t s) ;
virtual uint32_t getQueueSize() ;
virtual void setQueueSize(uint32_t s) ;
virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove queue_mv);
@ -184,12 +173,13 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
***/
virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details);
virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags);
virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) ;
virtual uint32_t getType(void *ref,FileSearchFlags flags) ;
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags);
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags);
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags);
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
/***
* Utility Functions
@ -203,6 +193,7 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
/***
* Directory Handling
***/
virtual void requestDirUpdate(void *ref) ; // triggers the update of the given reference. Used when browsing.
virtual void setDownloadDirectory(std::string path);
virtual void setPartialsDirectory(std::string path);
virtual std::string getDownloadDirectory();
@ -217,17 +208,10 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
virtual bool getShareDownloadDirectory();
virtual bool shareDownloadDirectory(bool share);
virtual void setRememberHashFilesDuration(uint32_t days) ;
virtual uint32_t rememberHashFilesDuration() const ;
virtual bool rememberHashFiles() const ;
virtual void setRememberHashFiles(bool) ;
virtual void clearHashCache() ;
virtual void setWatchPeriod(int minutes) ;
virtual int watchPeriod() const ;
/***************************************************************/
/*************** Control Interface *****************************/
/***************************************************************/
virtual void setWatchEnabled(bool b) ;
virtual bool watchEnabled() ;
/***************************************************************/
/*************** Data Transfer Interface ***********************/
@ -247,17 +231,6 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
bool addConfiguration(p3ConfigMgr *cfgmgr);
bool ResumeTransfers();
/******************* p3 Config Overload ************************/
protected:
/* Key Functions to be overloaded for Full Configuration */
virtual RsSerialiser *setupSerialiser();
virtual bool saveList(bool &cleanup, std::list<RsItem *>&);
virtual bool loadList(std::list<RsItem *>& load);
private:
bool loadConfigMap(std::map<std::string, std::string> &configMap);
/******************* p3 Config Overload ************************/
/*************************** p3 Config Overload ********************/
protected:
@ -277,17 +250,11 @@ class ftServer: public p3Service, public RsFiles, public ftDataSend, public RsTu
*/
p3PeerMgr *mPeerMgr;
p3ServiceControl *mServiceCtrl;
ftCacheStrapper *mCacheStrapper;
ftFiStore *mFiStore;
ftFiMonitor *mFiMon;
p3FileDatabase *mFileDatabase ;
ftController *mFtController;
ftExtraList *mFtExtra;
ftDataMultiplex *mFtDataplex;
p3turtle *mTurtleRouter ;
ftFileSearch *mFtSearch;
RsMutex srvMutex;

41
libretroshare/src/libretroshare.pro
View File

@ -37,6 +37,28 @@ debug {
QMAKE_CXXFLAGS *= -g -fno-omit-frame-pointer
}
CONFIG += file_lists
file_lists {
HEADERS *= file_sharing/p3filelists.h \
file_sharing/hash_cache.h \
file_sharing/filelist_io.h \
file_sharing/directory_storage.h \
file_sharing/directory_updater.h \
file_sharing/rsfilelistitems.h \
file_sharing/dir_hierarchy.h \
file_sharing/file_sharing_defaults.h
SOURCES *= file_sharing/p3filelists.cc \
file_sharing/hash_cache.cc \
file_sharing/filelist_io.cc \
file_sharing/directory_storage.cc \
file_sharing/directory_updater.cc \
file_sharing/dir_hierarchy.cc \
file_sharing/rsfilelistitems.cc
}
dsdv {
DEFINES *= SERVICES_DSDV
HEADERS += services/p3dsdv.h \
@ -345,16 +367,10 @@ INCLUDEPATH *= $${OPENPGPSDK_DIR}
PRE_TARGETDEPS *= $${OPENPGPSDK_DIR}/lib/libops.a
LIBS *= $${OPENPGPSDK_DIR}/lib/libops.a -lbz2
HEADERS += dbase/cachestrapper.h \
dbase/fimonitor.h \
dbase/findex.h \
dbase/fistore.h
HEADERS += ft/ftchunkmap.h \
ft/ftcontroller.h \
ft/ftdata.h \
ft/ftdatamultiplex.h \
ft/ftdbase.h \
ft/ftextralist.h \
ft/ftfilecreator.h \
ft/ftfileprovider.h \
@ -364,6 +380,10 @@ HEADERS += ft/ftchunkmap.h \
ft/fttransfermodule.h \
ft/ftturtlefiletransferitem.h
HEADERS += directory_updater.h \
directory_list.h \
p3filelists.h
HEADERS += chat/distantchat.h \
chat/p3chatservice.h \
chat/distributedchat.h \
@ -507,17 +527,9 @@ HEADERS += util/folderiterator.h \
util/rsscopetimer.h \
util/stacktrace.h
SOURCES += dbase/cachestrapper.cc \
dbase/fimonitor.cc \
dbase/findex.cc \
dbase/fistore.cc \
dbase/rsexpr.cc
SOURCES += ft/ftchunkmap.cc \
ft/ftcontroller.cc \
ft/ftdatamultiplex.cc \
ft/ftdbase.cc \
ft/ftextralist.cc \
ft/ftfilecreator.cc \
ft/ftfileprovider.cc \
@ -638,6 +650,7 @@ SOURCES += turtle/p3turtle.cc \
SOURCES += util/folderiterator.cc \
util/rsdebug.cc \
util/rsexpr.cc \
util/rscompress.cc \
util/smallobject.cc \
util/rsdir.cc \

16
libretroshare/src/plugins/pluginmanager.cc
View File

@ -17,7 +17,6 @@
#include <util/rsversioninfo.h>
#include <util/folderiterator.h>
#include <ft/ftserver.h>
#include <dbase/cachestrapper.h>
#include <retroshare/rsplugin.h>
#include <retroshare/rsfiles.h>
#include <pqi/pqiservice.h>
@ -146,7 +145,7 @@ void RsPluginManager::loadPlugins(const std::vector<std::string>& plugin_directo
continue ;
}
while(dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
std::string fname;
dirIt.d_name(fname);
@ -410,14 +409,17 @@ void RsPluginManager::slowTickPlugins(time_t seconds)
void RsPluginManager::registerCacheServices()
{
// this is removed since the old cache syste is gone, but we need to make it register new GXS group services instead.
#ifdef REMOVED
std::cerr << " Registering cache services." << std::endl;
for(uint32_t i=0;i<_plugins.size();++i)
if(_plugins[i].plugin != NULL && _plugins[i].plugin->rs_cache_service() != NULL)
{
rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ;
//rsFiles->getCacheStrapper()->addCachePair(CachePair(_plugins[i].plugin->rs_cache_service(),_plugins[i].plugin->rs_cache_service(),CacheId(_plugins[i].plugin->rs_service_id(), 0))) ;
std::cerr << " adding new cache pair for plugin " << _plugins[i].plugin->getPluginName() << ", with RS_ID " << _plugins[i].plugin->rs_service_id() << std::endl ;
}
#endif
}
void RsPluginManager::registerClientServices(p3ServiceServer *pqih)
@ -546,14 +548,6 @@ bool RsPluginManager::saveList(bool& cleanup, std::list<RsItem*>& list)
return true;
}
// RsCacheService::RsCacheService(uint16_t service_type,uint32_t tick_delay, RsPluginHandler* pgHandler)
// : CacheSource(service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getLocalCacheDir()),
// CacheStore (service_type, true, pgHandler->getFileServer()->getCacheStrapper(), pgHandler->getFileServer()->getCacheTransfer(), pgHandler->getRemoteCacheDir()),
// p3Config(),
// _tick_delay_in_seconds(tick_delay)
// {
// }
RsPQIService::RsPQIService(uint16_t /*service_type*/, uint32_t /*tick_delay_in_seconds*/, RsPluginHandler* /*pgHandler*/)
: p3Service(),p3Config()
{

3
libretroshare/src/plugins/rscacheservice.h
View File

@ -1,6 +1,5 @@
#pragma once
#include <dbase/cachestrapper.h>
#include "plugins/pluginmanager.h"
// This code needs to be re-written to work with GXS. For now it is obsolete.
@ -9,7 +8,7 @@
// The following class abstracts the construction of a cache service. The user only has to
// supply RS with a type ID. If the ID is already in use, RS will complain.
//
class RsCacheService: public CacheSource, public CacheStore, public p3Config
class RsCacheService: /* public CacheSource, public CacheStore, */ public p3Config
{
public:
RsCacheService(uint16_t type,uint32_t tick_delay_in_seconds, RsPluginHandler* pgHandler) ;

48
libretroshare/src/retroshare/rsexpr.h
View File

@ -1,6 +1,3 @@
#ifndef RS_EXPRESSIONS_H
#define RS_EXPRESSIONS_H
/*
* rs-core/src/rsiface: rsexpr.h
*
@ -26,16 +23,19 @@
*
*/
#pragma once
#include <string>
#include <list>
#include <stdint.h>
#include "retroshare/rstypes.h"
/******************************************************************************************
Enumerations defining the Operators usable in the Boolean search expressions
******************************************************************************************/
namespace RsRegularExpression
{
enum LogicalOperator{
AndOp=0, /* exp AND exp */
OrOp=1, /* exp OR exp */
@ -50,7 +50,7 @@ enum StringOperator{
EqualsString = 2 /* exactly equal*/
};
/*Relational operators ( >, <, >=, <=, == and InRange )*/
/* Comparison operators ( >, <, >=, <=, == and InRange ) */
enum RelOperator{
Equals = 0,
GreaterEquals = 1,
@ -105,25 +105,37 @@ classes:
******************************************************************************************/
class FileEntry;
/*!
* \brief The ExpFileEntry class
* This is the base class the regular expressions can operate on. Derive it to fit your own needs!
*/
class ExpFileEntry
{
public:
virtual const std::string& file_name() const =0;
virtual uint64_t file_size() const =0;
virtual time_t file_modtime() const =0;
virtual uint32_t file_popularity() const =0;
virtual std::string file_parent_path() const =0;
virtual const RsFileHash& file_hash() const =0;
};
class Expression
{
public:
virtual bool eval (FileEntry *file) = 0;
virtual bool eval (const ExpFileEntry& file) = 0;
virtual ~Expression() {};
virtual void linearize(LinearizedExpression& e) const = 0 ;
};
class CompoundExpression : public Expression
{
public:
CompoundExpression( enum LogicalOperator op, Expression * exp1, Expression *exp2)
: Lexp(exp1), Rexp(exp2), Op(op){ }
bool eval (FileEntry *file) {
bool eval (const ExpFileEntry& file) {
if (Lexp == NULL or Rexp == NULL) {
return false;
}
@ -231,7 +243,7 @@ class NameExpression: public StringExpression
public:
NameExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -244,7 +256,7 @@ class PathExpression: public StringExpression {
public:
PathExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -257,7 +269,7 @@ class ExtExpression: public StringExpression {
public:
ExtExpression(enum StringOperator op, std::list<std::string> &t, bool ic):
StringExpression(op,t,ic) {}
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -270,7 +282,7 @@ class HashExpression: public StringExpression {
public:
HashExpression(enum StringOperator op, std::list<std::string> &t):
StringExpression(op,t, true) {}
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -290,7 +302,7 @@ class DateExpression: public RelExpression<int>
DateExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
DateExpression(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -305,7 +317,7 @@ class SizeExpression: public RelExpression<int>
SizeExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
SizeExpression(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -320,7 +332,7 @@ class SizeExpressionMB: public RelExpression<int>
SizeExpressionMB(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
SizeExpressionMB(enum RelOperator op, int lv, int hv):
RelExpression<int>(op,lv,hv) {}
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -334,7 +346,7 @@ class PopExpression: public RelExpression<int>
PopExpression(enum RelOperator op, int v): RelExpression<int>(op,v,v){}
PopExpression(enum RelOperator op, int lv, int hv): RelExpression<int>(op,lv,hv) {}
PopExpression(const LinearizedExpression& e) ;
bool eval(FileEntry *file);
bool eval(const ExpFileEntry& file);
virtual void linearize(LinearizedExpression& e) const
{
@ -342,6 +354,6 @@ class PopExpression: public RelExpression<int>
RelExpression<int>::linearize(e) ;
}
};
}
#endif /* RS_EXPRESSIONS_H */

30
libretroshare/src/retroshare/rsfiles.h
View File

@ -36,9 +36,7 @@
class RsFiles;
extern RsFiles *rsFiles;
class Expression;
class CacheStrapper ;
class CacheTransfer;
namespace RsRegularExpression { class Expression; }
/* These are used mainly by ftController at the moment */
const uint32_t RS_FILE_CTRL_PAUSE = 0x00000100;
@ -65,7 +63,7 @@ const uint32_t RS_FILE_PEER_OFFLINE = 0x00002000;
// Flags used when requesting info about transfers, mostly to filter out the result.
//
const FileSearchFlags RS_FILE_HINTS_CACHE ( 0x00000001 );
const FileSearchFlags RS_FILE_HINTS_CACHE_deprecated ( 0x00000001 );
const FileSearchFlags RS_FILE_HINTS_EXTRA ( 0x00000002 );
const FileSearchFlags RS_FILE_HINTS_LOCAL ( 0x00000004 );
const FileSearchFlags RS_FILE_HINTS_REMOTE ( 0x00000008 );
@ -81,7 +79,7 @@ const FileSearchFlags RS_FILE_HINTS_PERMISSION_MASK ( 0x00000180 );// OR
//
const TransferRequestFlags RS_FILE_REQ_ANONYMOUS_ROUTING ( 0x00000040 ); // Use to ask turtle router to download the file.
const TransferRequestFlags RS_FILE_REQ_ASSUME_AVAILABILITY ( 0x00000200 ); // Assume full source availability. Used for cache files.
const TransferRequestFlags RS_FILE_REQ_CACHE ( 0x00000400 ); // Assume full source availability. Used for cache files.
const TransferRequestFlags RS_FILE_REQ_CACHE_deprecated ( 0x00000400 ); // Assume full source availability. Used for cache files.
const TransferRequestFlags RS_FILE_REQ_EXTRA ( 0x00000800 );
const TransferRequestFlags RS_FILE_REQ_MEDIA ( 0x00001000 );
const TransferRequestFlags RS_FILE_REQ_BACKGROUND ( 0x00002000 ); // To download slowly.
@ -147,8 +145,6 @@ class RsFiles
/***
* Control of Downloads Priority.
***/
virtual uint32_t getMinPrioritizedTransfers() = 0 ;
virtual void setMinPrioritizedTransfers(uint32_t s) = 0 ;
virtual uint32_t getQueueSize() = 0 ;
virtual void setQueueSize(uint32_t s) = 0 ;
virtual bool changeQueuePosition(const RsFileHash& hash, QueueMove mv) = 0;
@ -186,12 +182,13 @@ class RsFiles
*/
virtual int RequestDirDetails(const RsPeerId& uid, const std::string& path, DirDetails &details) = 0;
virtual int RequestDirDetails(void *ref, DirDetails &details, FileSearchFlags flags) = 0;
virtual bool findChildPointer(void *ref, int row, void *& result, FileSearchFlags flags) =0;
virtual uint32_t getType(void *ref,FileSearchFlags flags) = 0;
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags) = 0;
virtual int SearchKeywords(std::list<std::string> keywords, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0;
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags) = 0;
virtual int SearchBoolExp(Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0;
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags) = 0;
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0;
/***
* Utility Functions.
@ -205,6 +202,8 @@ class RsFiles
/***
* Directory Control
***/
virtual void requestDirUpdate(void *ref) =0 ; // triggers the update of the given reference. Used when browsing.
virtual void setDownloadDirectory(std::string path) = 0;
virtual void setPartialsDirectory(std::string path) = 0;
virtual std::string getDownloadDirectory() = 0;
@ -214,16 +213,11 @@ class RsFiles
virtual bool addSharedDirectory(const SharedDirInfo& dir) = 0;
virtual bool updateShareFlags(const SharedDirInfo& dir) = 0; // updates the flags. The directory should already exist !
virtual bool removeSharedDirectory(std::string dir) = 0;
virtual void setRememberHashFilesDuration(uint32_t days) = 0 ;
virtual uint32_t rememberHashFilesDuration() const = 0 ;
virtual void clearHashCache() = 0 ;
virtual bool rememberHashFiles() const =0;
virtual void setRememberHashFiles(bool) =0;
virtual void setWatchPeriod(int minutes) =0;
virtual int watchPeriod() const =0;
virtual CacheStrapper *getCacheStrapper() =0;
virtual CacheTransfer *getCacheTransfer() =0;
virtual void setWatchPeriod(int minutes) =0;
virtual void setWatchEnabled(bool b) =0;
virtual int watchPeriod() const =0;
virtual bool watchEnabled() =0;
virtual bool getShareDownloadDirectory() = 0;
virtual bool shareDownloadDirectory(bool share) = 0;

4
libretroshare/src/retroshare/rsturtle.h
View File

@ -35,7 +35,7 @@
#include "retroshare/rstypes.h"
class LinearizedExpression ;
namespace RsRegularExpression { class LinearizedExpression ; }
class RsTurtleClientService ;
class RsTurtle;
@ -102,7 +102,7 @@ class RsTurtle
// as they come back.
//
virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ;
virtual TurtleRequestId turtleSearch(const LinearizedExpression& expr) = 0 ;
virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ;
// Initiates tunnel handling for the given file hash. tunnels. Launches
// an exception if an error occurs during the initialization process. The

8
libretroshare/src/retroshare/rstypes.h
View File

@ -138,6 +138,7 @@ public:
/********************** For FileCache Interface *****************/
#define DIR_TYPE_UNKNOWN 0x00
#define DIR_TYPE_ROOT 0x01
#define DIR_TYPE_PERSON 0x02
#define DIR_TYPE_DIR 0x04
@ -216,7 +217,6 @@ class FileInfo
std::ostream &operator<<(std::ostream &out, const FileInfo& info);
class DirStub
{
public:
@ -236,16 +236,18 @@ class DirDetails
RsPeerId id;
std::string name;
RsFileHash hash;
std::string path;
std::string path; // full path of the parent directory, when it is a file; full path of the dir otherwise.
uint64_t count;
uint32_t age;
FileStorageFlags flags;
uint32_t min_age ; // minimum age of files in this subtree
std::list<DirStub> children;
std::vector<DirStub> children;
std::list<RsNodeGroupId> parent_groups; // parent groups for the shared directory
};
std::ostream &operator<<(std::ostream &out, const DirDetails& details);
class FileDetail
{
public:

2
libretroshare/src/rsserver/rsaccounts.cc
View File

@ -533,7 +533,7 @@ bool RsAccountsDetail::getAvailableAccounts(std::map<RsPeerId, AccountDetails> &
struct stat64 buf;
while (dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
/* check entry type */
std::string fname;

20
libretroshare/src/rsserver/rsinit.cc
View File

@ -56,7 +56,8 @@
#include <openssl/rand.h>
#include <fcntl.h>
#include <gxstunnel/p3gxstunnel.h>
#include "gxstunnel/p3gxstunnel.h"
#include "file_sharing/p3filelists.h"
#define ENABLE_GROUTER
@ -804,7 +805,6 @@ bool RsInit::SetHiddenLocation(const std::string& hiddenaddress, uint16_t port)
#include <unistd.h>
//#include <getopt.h>
#include "dbase/cachestrapper.h"
#include "ft/ftserver.h"
#include "ft/ftcontroller.h"
@ -1243,8 +1243,6 @@ int RsServer::StartupRetroShare()
ftserver->setConfigDirectory(rsAccounts->PathAccountDirectory());
ftserver->SetupFtServer() ;
CacheStrapper *mCacheStrapper = ftserver->getCacheStrapper();
//CacheTransfer *mCacheTransfer = ftserver->getCacheTransfer();
/* setup any extra bits (Default Paths) */
ftserver->setPartialsDirectory(emergencyPartialsDir);
@ -1288,7 +1286,6 @@ int RsServer::StartupRetroShare()
// These are needed to load plugins: plugin devs might want to know the place of
// cache directories, get pointers to cache strapper, or access ownId()
//
mPluginsManager->setCacheDirectories(localcachedir,remotecachedir) ;
mPluginsManager->setServiceControl(serviceCtrl) ;
// std::cerr << "rsinitconf (core 1) = " << (void*)rsInitConfig<<std::endl;
@ -1499,9 +1496,11 @@ int RsServer::StartupRetroShare()
pqih->addService(gr,true) ;
#endif
p3FileDatabase *fdb = new p3FileDatabase(serviceCtrl) ;
p3turtle *tr = new p3turtle(serviceCtrl,mLinkMgr) ;
rsTurtle = tr ;
pqih -> addService(tr,true);
pqih -> addService(fdb,true);
pqih -> addService(ftserver,true);
mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ;
@ -1514,6 +1513,7 @@ int RsServer::StartupRetroShare()
// connect components to turtle router.
ftserver->connectToTurtleRouter(tr) ;
ftserver->connectToFileDatabase(fdb) ;
chatSrv->connectToGxsTunnelService(mGxsTunnels) ;
gr->connectToTurtleRouter(tr) ;
#ifdef ENABLE_GROUTER
@ -1619,10 +1619,6 @@ int RsServer::StartupRetroShare()
mLinkMgr->addMonitor(serviceCtrl);
mLinkMgr->addMonitor(serviceInfo);
// NOTE these were added in ftServer (was added twice).
//mLinkMgr->addMonitor(mCacheStrapper);
//mLinkMgr->addMonitor(((ftController *) mCacheTransfer));
// Services that have been changed to pqiServiceMonitor
serviceCtrl->registerServiceMonitor(msgSrv, msgSrv->getServiceInfo().mServiceType);
serviceCtrl->registerServiceMonitor(mDisc, mDisc->getServiceInfo().mServiceType);
@ -1678,12 +1674,6 @@ int RsServer::StartupRetroShare()
/**************************************************************************/
std::cerr << "(2) Load configuration files" << std::endl;
/* NOTE: CacheStrapper's load causes Cache Files to be
* loaded into all the CacheStores/Sources. This happens
* after all the other configurations have happened.
*/
mConfigMgr->addConfiguration("cache.cfg", mCacheStrapper);
mConfigMgr->loadConfiguration();
/**************************************************************************/

21
libretroshare/src/rsserver/rstypes.cc
View File

@ -44,7 +44,26 @@
*
**********************************************************************/
std::ostream &operator<<(std::ostream &out, const DirDetails& d)
{
std::cerr << "====DIR DETAILS====" << std::endl;
std::cerr << " parent pointer: " << d.parent << std::endl;
std::cerr << " current pointer: " << d.ref << std::endl;
std::cerr << " parent row : " << d.prow << std::endl;
std::cerr << " type : " << (int)d.type << std::endl;
std::cerr << " PeerId : " << d.id << std::endl;
std::cerr << " Name : " << d.name << std::endl;
std::cerr << " Hash : " << d.hash << std::endl;
std::cerr << " Path : " << d.path << std::endl;
std::cerr << " Count : " << d.count << std::endl;
std::cerr << " Age : " << d.age << std::endl;
std::cerr << " Min age : " << d.min_age << std::endl;
std::cerr << " Flags : " << d.flags << std::endl;
std::cerr << " Parent groups : " ; for(std::list<RsNodeGroupId>::const_iterator it(d.parent_groups.begin());it!=d.parent_groups.end();++it) std::cerr << (*it) << " "; std::cerr << std::endl;
std::cerr << " Children : " ; for(uint32_t i=0;i<d.children.size();++i) std::cerr << (void*)(intptr_t)d.children[i].ref << " "; std::cerr << std::endl;
std::cerr << "===================" << std::endl;
return out;
}
std::ostream &operator<<(std::ostream &out, const FileInfo &info)
{
out << "FileInfo: path: " << info.path;

5
libretroshare/src/serialiser/itempriorities.h
View File

@ -76,6 +76,11 @@ const uint8_t QOS_PRIORITY_RS_DISC_PGP_LIST = 2 ; // same priority.
const uint8_t QOS_PRIORITY_RS_DISC_SERVICES = 2 ;
const uint8_t QOS_PRIORITY_RS_DISC_PGP_CERT = 1 ;
// File database
//
const uint8_t QOS_PRIORITY_RS_FAST_SYNC_REQUEST = 7 ;
const uint8_t QOS_PRIORITY_RS_SLOW_SYNC_REQUEST = 3 ;
// Heartbeat.
//
const uint8_t QOS_PRIORITY_RS_HEARTBEAT_PULSE = 8 ;

3
libretroshare/src/serialiser/rsbaseserial.cc
View File

@ -231,6 +231,9 @@ uint32_t getRawStringSize(const std::string &outStr)
bool getRawString(void *data, uint32_t size, uint32_t *offset, std::string &outStr)
{
#warning I had to change this. It seems like a bug to not clear the string. Should make sure it's not introducing any side effect.
outStr.clear();
uint32_t len = 0;
if (!getRawUInt32(data, size, offset, &len))
{

1
libretroshare/src/serialiser/rsserviceids.h
View File

@ -49,6 +49,7 @@ const uint16_t RS_SERVICE_TYPE_TUNNEL = 0x0015;
const uint16_t RS_SERVICE_TYPE_HEARTBEAT = 0x0016;
const uint16_t RS_SERVICE_TYPE_FILE_TRANSFER = 0x0017;
const uint16_t RS_SERVICE_TYPE_GROUTER = 0x0018;
const uint16_t RS_SERVICE_TYPE_FILE_DATABASE = 0x0019;
const uint16_t RS_SERVICE_TYPE_SERVICEINFO = 0x0020;
/* Bandwidth Control */

4
libretroshare/src/turtle/p3turtle.cc
View File

@ -1763,7 +1763,7 @@ void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
std::list<DirDetails> initialResults;
// to do: split search string into words.
Expression *exp = LinearizedExpression::toExpr(expr) ;
RsRegularExpression::Expression *exp = RsRegularExpression::LinearizedExpression::toExpr(expr) ;
if(exp == NULL)
return ;
@ -1820,7 +1820,7 @@ TurtleRequestId p3turtle::turtleSearch(const std::string& string_to_match)
return id ;
}
TurtleRequestId p3turtle::turtleSearch(const LinearizedExpression& expr)
TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpression& expr)
{
// generate a new search id.

2
libretroshare/src/turtle/p3turtle.h
View File

@ -239,7 +239,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
// remove the specific file search packets from the turtle router.
//
virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
virtual TurtleSearchRequestId turtleSearch(const LinearizedExpression& expr) ;
virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
// Initiates tunnel handling for the given file hash. tunnels. Launches
// an exception if an error occurs during the initialization process. The

2
libretroshare/src/turtle/rsturtleitem.h
View File

@ -102,7 +102,7 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem
RsTurtleRegExpSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {}
RsTurtleRegExpSearchRequestItem(void *data,uint32_t size) ;
LinearizedExpression expr ; // Reg Exp in linearised mode
RsRegularExpression::LinearizedExpression expr ; // Reg Exp in linearised mode
virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; }
virtual void performLocalSearch(std::list<TurtleFileInfo>&) const ;

115
libretroshare/src/util/folderiterator.cc
View File

@ -1,12 +1,39 @@
#include <dirent.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <unistd.h>
#ifdef WINDOWS_SYS
#include "util/rswin.h"
#endif
#include "folderiterator.h"
#include "rsstring.h"
namespace librs { namespace util {
FolderIterator::FolderIterator(const std::string& folderName)
: mFolderName(folderName)
{
// Grab the last modification time for the directory
struct stat64 buf ;
#ifdef WINDOWS_SYS
std::wstring wfullname;
librs::util::ConvertUtf8ToUtf16(folderName, wfullname);
if ( 0 == _wstati64(wfullname.c_str(), &buf))
#else
if ( 0 == stat64(folderName.c_str(), &buf))
#endif
{
mFolderModTime = buf.st_mtime ;
}
// Now open directory content and read the first entry
#ifdef WINDOWS_SYS
std::wstring utf16Name;
if(! ConvertUtf8ToUtf16(folderName, utf16Name)) {
@ -17,11 +44,11 @@ FolderIterator::FolderIterator(const std::string& folderName)
utf16Name += L"/*.*";
handle = FindFirstFileW(utf16Name.c_str(), &fileInfo);
validity = handle != INVALID_HANDLE_VALUE;
isFirstCall = true;
is_open = validity = handle != INVALID_HANDLE_VALUE;
#else
handle = opendir(folderName.c_str());
validity = handle != NULL;
is_open = validity = handle != NULL;
next();
#endif
}
@ -30,19 +57,71 @@ FolderIterator::~FolderIterator()
closedir();
}
bool FolderIterator::readdir() {
void FolderIterator::next()
{
do {
if(!readdir())
{
validity = false ;
break ;
}
d_name(mFileName);
} while(mFileName == "." || mFileName == "..") ;
mFullPath = mFolderName + "/" + mFileName ;
struct stat64 buf ;
#ifdef WINDOWS_SYS
std::wstring wfullname;
librs::util::ConvertUtf8ToUtf16(mFullPath, wfullname);
if ( 0 == _wstati64(wfullname.c_str(), &buf))
#else
if ( 0 == stat64(mFullPath.c_str(), &buf))
#endif
{
mFileModTime = buf.st_mtime ;
mStatInfoOk = true;
if (S_ISDIR(buf.st_mode))
{
mType = TYPE_DIR ;
mFileSize = 0 ;
mFileModTime = buf.st_mtime;
}
else if (S_ISREG(buf.st_mode))
{
mType = TYPE_FILE ;
mFileSize = buf.st_size;
mFileModTime = buf.st_mtime;
}
else
{
mType = TYPE_UNKNOWN ;
mFileSize = 0 ;
mFileModTime = 0;
}
}
else
{
mType = TYPE_UNKNOWN ;
mFileSize = 0 ;
mFileModTime = 0;
validity = false ;
}
}
bool FolderIterator::readdir()
{
if(!validity)
return false;
#ifdef WINDOWS_SYS
if(isFirstCall) {
isFirstCall = false;
return true;
}
return FindNextFileW(handle, &fileInfo) != 0;
#else
ent = ::readdir(handle);
return ent != 0;
return ent != NULL;
#endif
}
@ -65,13 +144,23 @@ bool FolderIterator::d_name(std::string& dest)
return true;
}
time_t FolderIterator::dir_modtime() const { return mFolderModTime ; }
const std::string& FolderIterator::file_fullpath() { return mFullPath ; }
const std::string& FolderIterator::file_name() { return mFileName ; }
uint64_t FolderIterator::file_size() { return mFileSize ; }
time_t FolderIterator::file_modtime() { return mFileModTime ; }
uint8_t FolderIterator::file_type() { return mType ; }
bool FolderIterator::closedir()
{
if(!validity)
return false;
validity = false;
if(!is_open)
return true ;
is_open = false ;
#ifdef WINDOWS_SYS
return FindClose(handle) != 0;
#else

31
libretroshare/src/util/folderiterator.h
View File

@ -2,6 +2,7 @@
#define FOLDERITERATOR_H
#include <stdint.h>
#include <iostream>
#include <cstdio>
@ -24,15 +25,32 @@ public:
FolderIterator(const std::string& folderName);
~FolderIterator();
enum { TYPE_UNKNOWN = 0x00,
TYPE_FILE = 0x01,
TYPE_DIR = 0x02
};
// info about current parent directory
time_t dir_modtime() const ;
// info about directory content
bool isValid() const { return validity; }
bool readdir();
void next();
#warning this one should go, as it reports the same information than file_name()
bool d_name(std::string& dest);
bool closedir();
const std::string& file_name() ;
const std::string& file_fullpath() ;
uint64_t file_size() ;
uint8_t file_type() ;
time_t file_modtime() ;
private:
bool is_open;
bool validity;
#ifdef WINDOWS_SYS
@ -43,7 +61,16 @@ private:
DIR* handle;
struct dirent* ent;
#endif
void updateStatsInfo() ;
bool mStatInfoOk ;
time_t mFileModTime ;
time_t mFolderModTime ;
uint64_t mFileSize ;
uint8_t mType ;
std::string mFileName ;
std::string mFullPath ;
std::string mFolderName ;
};

287
libretroshare/src/util/rsexpr.cc Normal file
View File

@ -0,0 +1,287 @@
/*
* rs-core/src/dbase: rsexpr.cc
*
* RetroShare C++ Interface.
*
* Copyright 2007-2008 by Kashif Kaleem.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "retroshare/rsexpr.h"
#include "retroshare/rstypes.h"
#include <algorithm>
#include <functional>
/******************************************************************************************
eval functions of relational expressions.
******************************************************************************************/
namespace RsRegularExpression
{
template<>
void RelExpression<int>::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(LowerValue) ;
e._ints.push_back(HigherValue) ;
}
bool DateExpression::eval(const ExpFileEntry& file)
{
return evalRel(file.file_modtime());
}
bool SizeExpressionMB::eval(const ExpFileEntry& file)
{
return evalRel((int)(file.file_size()/(uint64_t)(1024*1024)));
}
bool SizeExpression::eval(const ExpFileEntry& file)
{
return evalRel(file.file_size());
}
bool PopExpression::eval(const ExpFileEntry& file)
{
return evalRel(file.file_popularity());
}
/******************************************************************************************
Code for evaluating string expressions
******************************************************************************************/
bool NameExpression::eval(const ExpFileEntry& file)
{
return evalStr(file.file_name());
}
bool PathExpression::eval(const ExpFileEntry& file)
{
return evalStr(file.file_parent_path());
}
bool ExtExpression::eval(const ExpFileEntry& file)
{
std::string ext;
/*Get the part of the string after the last instance of . in the filename */
size_t index = file.file_name().find_last_of('.');
if (index != std::string::npos) {
ext = file.file_name().substr(index+1);
if (ext != "" ){
return evalStr(ext);
}
}
return false;
}
bool HashExpression::eval(const ExpFileEntry& file){
return evalStr(file.file_hash().toStdString());
}
/*Check whether two strings are 'equal' to each other*/
static bool StrEquals(const std::string & str1, const std::string & str2,
bool IgnoreCase ){
if ( str1.size() != str2.size() ){
return false;
} else if (IgnoreCase) {
std::equal( str1.begin(), str1.end(),
str2.begin(), CompareCharIC() );
}
return std::equal( str1.begin(), str1.end(),
str2.begin());
}
/*Check whether one string contains the other*/
static bool StrContains( const std::string & str1, const std::string & str2,
bool IgnoreCase){
std::string::const_iterator iter ;
if (IgnoreCase) {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end(), CompareCharIC() );
} else {
iter = std::search( str1.begin(), str1.end(),
str2.begin(), str2.end());
}
return ( iter != str1.end() );
}
bool StringExpression :: evalStr ( const std::string &str ){
std::list<std::string>::iterator iter;
switch (Op) {
case ContainsAllStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str, *iter, IgnoreCase) == false ){
return false;
}
}
return true;
break;
case ContainsAnyStrings:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrContains (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
case EqualsString:
for ( iter = terms.begin(); iter != terms.end(); ++iter ) {
if ( StrEquals (str,*iter, IgnoreCase) == true ) {
return true;
}
}
break;
default:
return false;
}
return false;
}
/*************************************************************************
* linearization code
*************************************************************************/
void CompoundExpression::linearize(LinearizedExpression& e) const
{
e._tokens.push_back(LinearizedExpression::EXPR_COMP) ;
e._ints.push_back(Op) ;
Lexp->linearize(e) ;
Rexp->linearize(e) ;
}
void StringExpression::linearize(LinearizedExpression& e) const
{
e._ints.push_back(Op) ;
e._ints.push_back(IgnoreCase) ;
e._ints.push_back(terms.size()) ;
for(std::list<std::string>::const_iterator it(terms.begin());it!=terms.end();++it)
e._strings.push_back(*it) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e)
{
int i=0,j=0,k=0 ;
return toExpr(e,i,j,k) ;
}
void LinearizedExpression::readStringExpr(const LinearizedExpression& e,int& n_ints,int& n_strings,std::list<std::string>& strings,bool& b,StringOperator& op)
{
op = static_cast<StringOperator>(e._ints[n_ints++]) ;
b = e._ints[n_ints++] ;
int n = e._ints[n_ints++] ;
strings.clear() ;
for(int i=0;i<n;++i)
strings.push_back(e._strings[n_strings++]) ;
}
Expression *LinearizedExpression::toExpr(const LinearizedExpression& e,int& n_tok,int& n_ints,int& n_strings)
{
LinearizedExpression::token tok = static_cast<LinearizedExpression::token>(e._tokens[n_tok++]) ;
switch(tok)
{
case EXPR_DATE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new DateExpression(op,lv,hv) ;
}
case EXPR_POP: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new PopExpression(op,lv,hv) ;
}
case EXPR_SIZE: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpression(op,lv,hv) ;
}
case EXPR_HASH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new HashExpression(op,strings) ;
}
case EXPR_NAME: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new NameExpression(op,strings,b) ;
}
case EXPR_PATH: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_EXT: {
std::list<std::string> strings ;
StringOperator op ;
bool b ;
readStringExpr(e,n_ints,n_strings,strings,b,op) ;
return new ExtExpression(op,strings,b) ;
}
case EXPR_COMP: {
LogicalOperator op = static_cast<LogicalOperator>(e._ints[n_ints++]) ;
Expression *e1 = toExpr(e,n_tok,n_ints,n_strings) ;
Expression *e2 = toExpr(e,n_tok,n_ints,n_strings) ;
return new CompoundExpression(op,e1,e2) ;
}
case EXPR_SIZE_MB: {
RelOperator op = static_cast<RelOperator>(e._ints[n_ints++]) ;
int lv = e._ints[n_ints++] ;
int hv = e._ints[n_ints++] ;
return new SizeExpressionMB(op,lv,hv) ;
}
default:
std::cerr << "No expression match the current value " << tok << std::endl ;
return NULL ;
}
}
}

13
retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp
View File

@ -774,7 +774,7 @@ void SearchDialog::initSearchResult(const QString& txt, qulonglong searchId, int
ui.searchSummaryWidget->setCurrentItem(item2);
}
void SearchDialog::advancedSearch(Expression* expression)
void SearchDialog::advancedSearch(RsRegularExpression::Expression* expression)
{
advSearchDialog->hide();
@ -782,7 +782,7 @@ void SearchDialog::advancedSearch(Expression* expression)
std::list<DirDetails> results;
// send a turtle search request
LinearizedExpression e ;
RsRegularExpression::LinearizedExpression e ;
expression->linearize(e) ;
TurtleRequestId req_id = rsTurtle->turtleSearch(e) ;
@ -843,8 +843,8 @@ void SearchDialog::searchKeywords(const QString& keywords)
if (n < 1)
return;
NameExpression exprs(ContainsAllStrings,words,true) ;
LinearizedExpression lin_exp ;
RsRegularExpression::NameExpression exprs(RsRegularExpression::ContainsAllStrings,words,true) ;
RsRegularExpression::LinearizedExpression lin_exp ;
exprs.linearize(lin_exp) ;
TurtleRequestId req_id ;
@ -1042,9 +1042,10 @@ void SearchDialog::insertDirectory(const QString &txt, qulonglong searchId, cons
}
/* go through all children directories/files for a recursive call */
for (std::list<DirStub>::const_iterator it(dir.children.begin()); it != dir.children.end(); ++it) {
for (uint32_t i=0;i<dir.children.size();++i)
{
DirDetails details;
rsFiles->RequestDirDetails(it->ref, details, FileSearchFlags(0u));
rsFiles->RequestDirDetails(dir.children[i].ref, details, FileSearchFlags(0u));
insertDirectory(txt, searchId, details, child);
}
}

5
retroshare-gui/src/gui/FileTransfer/SearchDialog.h
View File

@ -27,9 +27,10 @@
#include "mainpage.h"
class AdvancedSearchDialog;
class Expression;
class RSTreeWidgetItemCompareRole;
namespace RsRegularExpression { class Expression; }
#define FRIEND_SEARCH 1
#define ANONYMOUS_SEARCH 2
class SearchDialog : public MainPage
@ -99,7 +100,7 @@ private slots:
void showAdvSearchDialog(bool=true);
/** perform the advanced search */
void advancedSearch(Expression*);
void advancedSearch(RsRegularExpression::Expression*);
void selectSearchResults(int index = -1);
void hideOrShowSearchResult(QTreeWidgetItem* resultItem, QString currentSearchId = QString(), int fileTypeIndex = -1);

33
retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp
View File

@ -349,9 +349,6 @@ TransfersDialog::TransfersDialog(QWidget *parent)
// ui.tunnelInfoWidget->setFocusPolicy(Qt::NoFocus);
/** Setup the actions for the context menu */
toggleShowCacheTransfersAct = new QAction(tr( "Show file list transfers" ), this );
toggleShowCacheTransfersAct->setCheckable(true) ;
connect(toggleShowCacheTransfersAct,SIGNAL(triggered()),this,SLOT(toggleShowCacheTransfers())) ;
// Actions. Only need to be defined once.
pauseAct = new QAction(QIcon(IMAGE_PAUSE), tr("Pause"), this);
@ -519,12 +516,6 @@ UserNotify *TransfersDialog::getUserNotify(QObject *parent)
return new TransferUserNotify(parent);
}
void TransfersDialog::toggleShowCacheTransfers()
{
_show_cache_transfers = !_show_cache_transfers ;
insertTransfers() ;
}
void TransfersDialog::processSettings(bool bLoad)
{
m_bProcessSettings = true;
@ -537,9 +528,6 @@ void TransfersDialog::processSettings(bool bLoad)
if (bLoad) {
// load settings
// state of checks
_show_cache_transfers = Settings->value("showCacheTransfers", false).toBool();
// state of the lists
DLHeader->restoreState(Settings->value("downloadList").toByteArray());
ULHeader->restoreState(Settings->value("uploadList").toByteArray());
@ -565,9 +553,6 @@ void TransfersDialog::processSettings(bool bLoad)
} else {
// save settings
// state of checks
Settings->setValue("showCacheTransfers", _show_cache_transfers);
// state of the lists
Settings->setValue("downloadList", DLHeader->saveState());
Settings->setValue("uploadList", ULHeader->saveState());
@ -801,9 +786,6 @@ void TransfersDialog::downloadListCustomPopupMenu( QPoint /*point*/ )
contextMnu.addSeparator() ;//-----------------------------------------------
contextMnu.addAction( toggleShowCacheTransfersAct ) ;
toggleShowCacheTransfersAct->setChecked(_show_cache_transfers) ;
collCreateAct->setEnabled(true) ;
collModifAct->setEnabled(single && add_CollActions) ;
collViewAct->setEnabled(single && add_CollActions) ;
@ -1269,13 +1251,6 @@ void TransfersDialog::insertTransfers()
continue;
}
if ((fileInfo.transfer_info_flags & RS_FILE_REQ_CACHE) && !_show_cache_transfers) {
// if file transfer is a cache file index file, don't show it
DLListModel->removeRow(row);
rowCount = DLListModel->rowCount();
continue;
}
hashs.erase(hashIt);
if (addItem(row, fileInfo) < 0) {
@ -1295,11 +1270,6 @@ void TransfersDialog::insertTransfers()
continue;
}
if ((fileInfo.transfer_info_flags & RS_FILE_REQ_CACHE) && !_show_cache_transfers) {
//if file transfer is a cache file index file, don't show it
continue;
}
addItem(-1, fileInfo);
}
@ -1322,9 +1292,6 @@ void TransfersDialog::insertTransfers()
if (!rsFiles->FileDetails(*it, RS_FILE_HINTS_UPLOAD, info))
continue;
if((info.transfer_info_flags & RS_FILE_REQ_CACHE) && _show_cache_transfers)
continue ;
std::list<TransferInfo>::iterator pit;
for(pit = info.peers.begin(); pit != info.peers.end(); ++pit)
{

3
retroshare-gui/src/gui/FileTransfer/TransfersDialog.h
View File

@ -199,7 +199,6 @@ private:
QAction *chunkProgressiveAct;
QAction *chunkStreamingAct;
QAction *detailsFileAct;
QAction *toggleShowCacheTransfersAct;
QAction *renameFileAct;
QAction *specifyDestinationDirectoryAct;
QAction *expandAllAct;
@ -249,7 +248,6 @@ private:
/** Qt Designer generated object */
Ui::TransfersDialog ui;
bool _show_cache_transfers ;
public slots:
// these two functions add entries to the transfers dialog, and return the row id of the entry modified/added
//
@ -259,7 +257,6 @@ public slots:
int addUploadItem(const QString& symbol, const QString& name, const QString& coreID, qlonglong size, const FileProgressInfo& pinfo, double dlspeed, const QString& sources,const QString& source_id, const QString& status, qlonglong completed, qlonglong remaining);
void showFileDetails() ;
void toggleShowCacheTransfers() ;
double getProgress(int row, QStandardItemModel *model);
double getSpeed(int row, QStandardItemModel *model);

5
retroshare-gui/src/gui/MainWindow.cpp
View File

@ -756,7 +756,10 @@ void MainWindow::updateFriends()
void MainWindow::postModDirectories(bool update_local)
{
RSettingsWin::postModDirectories(update_local);
ShareManager::postModDirectories(update_local);
// Why would we need that?? The effect is to reset the flags while we're changing them, so it's really not
// a good idea.
//ShareManager::postModDirectories(update_local);
QCoreApplication::flush();
}

311
retroshare-gui/src/gui/RemoteDirModel.cpp
View File

@ -80,6 +80,11 @@ void RetroshareDirModel::treeStyle()
peerIcon = QIcon(":/images/user/identity16.png");
}
void TreeStyle_RDM::updateRef(const QModelIndex& indx) const
{
rsFiles->requestDirUpdate(indx.internalPointer()) ;
}
bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const
{
@ -99,9 +104,8 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const
void *ref = parent.internalPointer();
const DirDetails *details = requestDirDetails(ref, RemoteMode);
if (!details)
DirDetails details ;
if (!requestDirDetails(ref, RemoteMode,details))
{
/* error */
#ifdef RDM_DEBUG
@ -111,7 +115,7 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const
return false;
}
if (details->type == DIR_TYPE_FILE)
if (details.type == DIR_TYPE_FILE)
{
#ifdef RDM_DEBUG
std::cerr << "lookup FILE -> false";
@ -124,7 +128,7 @@ bool TreeStyle_RDM::hasChildren(const QModelIndex &parent) const
std::cerr << "lookup PER/DIR #" << details->count;
std::cerr << std::endl;
#endif
return (details->count > 0); /* do we have children? */
return (details.count > 0); /* do we have children? */
}
bool FlatStyle_RDM::hasChildren(const QModelIndex &parent) const
{
@ -155,9 +159,9 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const
void *ref = (parent.isValid())? parent.internalPointer() : NULL ;
const DirDetails *details = requestDirDetails(ref, RemoteMode);
DirDetails details ;
if (!details)
if (! requestDirDetails(ref, RemoteMode,details))
{
#ifdef RDM_DEBUG
std::cerr << "lookup failed -> 0";
@ -165,7 +169,7 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const
#endif
return 0;
}
if (details->type == DIR_TYPE_FILE)
if (details.type == DIR_TYPE_FILE)
{
#ifdef RDM_DEBUG
std::cerr << "lookup FILE: 0";
@ -179,7 +183,7 @@ int TreeStyle_RDM::rowCount(const QModelIndex &parent) const
std::cerr << "lookup PER/DIR #" << details->count;
std::cerr << std::endl;
#endif
return details->count;
return details.count;
}
int FlatStyle_RDM::rowCount(const QModelIndex &parent) const
@ -190,6 +194,7 @@ int FlatStyle_RDM::rowCount(const QModelIndex &parent) const
std::cerr << "RetroshareDirModel::rowCount(): " << parent.internalPointer();
std::cerr << ": ";
#endif
RS_STACK_MUTEX(_ref_mutex) ;
return _ref_entries.size() ;
}
@ -294,6 +299,9 @@ QVariant RetroshareDirModel::decorationRole(const DirDetails& details,int coln)
else if (details.type == DIR_TYPE_FILE) /* File */
{
// extensions predefined
if(details.hash.isNull())
return QIcon(":/images/reset.png") ; // file is being hashed
else
return FilesDefs::getIconFromFilename(QString::fromUtf8(details.name.c_str()));
}
else
@ -381,16 +389,16 @@ QVariant TreeStyle_RDM::displayRole(const DirDetails& details,int coln) const
QString FlatStyle_RDM::computeDirectoryPath(const DirDetails& details) const
{
QString dir ;
const DirDetails *det = requestDirDetails(details.parent,RemoteMode);
DirDetails det ;
if(!det)
if(!requestDirDetails(details.parent,RemoteMode,det))
return QString();
#ifdef SHOW_TOTAL_PATH
do
{
#endif
dir = QString::fromUtf8(det->name.c_str())+"/"+dir ;
dir = QString::fromUtf8(det.name.c_str())+"/"+dir ;
#ifdef SHOW_TOTAL_PATH
if(!requestDirDetails(det.parent,det,flags))
@ -497,7 +505,12 @@ QVariant FlatStyle_RDM::sortRole(const QModelIndex& index,const DirDetails& deta
case 1: return (qulonglong) details.count;
case 2: return details.age;
case 3: return QString::fromUtf8(rsPeers->getPeerName(details.id).c_str());
case 4: return _ref_entries[index.row()].second ;
case 4: {
RS_STACK_MUTEX(_ref_mutex) ;
return computeDirectoryPath(details);
}
}
}
return QVariant();
@ -521,28 +534,30 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const
void *ref = index.internalPointer();
int coln = index.column();
const DirDetails *details = requestDirDetails(ref, RemoteMode);
DirDetails details ;
if (!details)
if (!requestDirDetails(ref, RemoteMode,details))
return QVariant();
if (role == RetroshareDirModel::FileNameRole) /* end of FileNameRole */
return QString::fromUtf8(details->name.c_str());
return QString::fromUtf8(details.name.c_str());
if (role == Qt::TextColorRole)
{
if(details->min_age > ageIndicator)
if(details.type == DIR_TYPE_FILE && details.hash.isNull())
return QVariant(QColor(Qt::green)) ;
else if(details.min_age > ageIndicator)
return QVariant(QColor(Qt::gray)) ;
else if(RemoteMode)
{
FileInfo info;
QVariant local_file_color = QVariant(QColor(Qt::red));
if(rsFiles->alreadyHaveFile(details->hash, info))
if(rsFiles->alreadyHaveFile(details.hash, info))
return local_file_color;
std::list<RsFileHash> downloads;
rsFiles->FileDownloads(downloads);
if(std::find(downloads.begin(), downloads.end(), details->hash) != downloads.end())
if(std::find(downloads.begin(), downloads.end(), details.hash) != downloads.end())
return local_file_color;
else
return QVariant();
@ -553,7 +568,7 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const
if(role == Qt::DecorationRole)
return decorationRole(*details,coln) ;
return decorationRole(details,coln) ;
/*****************
Qt::EditRole
@ -578,10 +593,10 @@ QVariant RetroshareDirModel::data(const QModelIndex &index, int role) const
} /* end of TextAlignmentRole */
if (role == Qt::DisplayRole)
return displayRole(*details,coln) ;
return displayRole(details,coln) ;
if (role == SortRole)
return sortRole(index,*details,coln) ;
return sortRole(index,details,coln) ;
return QVariant();
}
@ -700,6 +715,9 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent
std::cerr << ": row:" << row << " col:" << column << " ";
#endif
// This function is used extensively. There's no way we can use requestDirDetails() in it, which would
// cause far too much overhead. So we use a dedicated function that only grabs the required information.
if(row < 0)
return QModelIndex() ;
@ -712,40 +730,15 @@ QModelIndex TreeStyle_RDM::index(int row, int column, const QModelIndex & parent
}
********/
const DirDetailsVector *details = requestDirDetails(ref, RemoteMode);
if (!details)
{
#ifdef RDM_DEBUG
std::cerr << "lookup failed -> invalid";
std::cerr << std::endl;
#endif
void *result ;
if(rsFiles->findChildPointer(ref, row, result, ((RemoteMode) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL)))
return createIndex(row, column, result) ;
else
return QModelIndex();
}
/* now iterate through the details to
* get the reference number
*/
if (row >= (int) details->childrenVector.size())
{
#ifdef RDM_DEBUG
std::cerr << "wrong number of children -> invalid";
std::cerr << std::endl;
#endif
return QModelIndex();
}
#ifdef RDM_DEBUG
std::cerr << "success index(" << row << "," << column << "," << details->childrenVector[row].ref << ")";
std::cerr << std::endl;
#endif
/* we can just grab the reference now */
return createIndex(row, column, details->childrenVector[row].ref);
}
QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent) const
{
Q_UNUSED(parent);
@ -757,10 +750,15 @@ QModelIndex FlatStyle_RDM::index(int row, int column, const QModelIndex & parent
if(row < 0)
return QModelIndex() ;
RS_STACK_MUTEX(_ref_mutex) ;
if(row < (int) _ref_entries.size())
{
void *ref = _ref_entries[row].first ;
void *ref = _ref_entries[row];
#ifdef RDM_DEBUG
std::cerr << "Creating index 2 row=" << row << ", column=" << column << ", ref=" << (void*)ref << std::endl;
#endif
return createIndex(row, column, ref);
}
else
@ -786,9 +784,9 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const
}
void *ref = index.internalPointer();
const DirDetails *details = requestDirDetails(ref, RemoteMode);
DirDetails details ;
if (!details)
if (! requestDirDetails(ref, RemoteMode,details))
{
#ifdef RDM_DEBUG
std::cerr << "Failed Lookup -> invalid";
@ -797,7 +795,7 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const
return QModelIndex();
}
if (!(details->parent))
if (!(details.parent))
{
#ifdef RDM_DEBUG
std::cerr << "success. parent is Root/NULL --> invalid";
@ -810,8 +808,9 @@ QModelIndex TreeStyle_RDM::parent( const QModelIndex & index ) const
std::cerr << "success index(" << details->prow << ",0," << details->parent << ")";
std::cerr << std::endl;
std::cerr << "Creating index 3 row=" << details.prow << ", column=" << 0 << ", ref=" << (void*)details.parent << std::endl;
#endif
return createIndex(details->prow, 0, details->parent);
return createIndex(details.prow, 0, details.parent);
}
QModelIndex FlatStyle_RDM::parent( const QModelIndex & index ) const
{
@ -836,12 +835,12 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const
void *ref = index.internalPointer();
const DirDetails *details = requestDirDetails(ref, RemoteMode);
DirDetails details ;
if (!details)
if (! requestDirDetails(ref, RemoteMode,details))
return Qt::ItemIsSelectable; // Error.
switch(details->type)
switch(details.type)
{
case DIR_TYPE_PERSON: return Qt::ItemIsEnabled;
case DIR_TYPE_DIR: return Qt::ItemIsSelectable | Qt::ItemIsEnabled;
@ -862,6 +861,7 @@ Qt::ItemFlags RetroshareDirModel::flags( const QModelIndex & index ) const
/* Callback from */
void RetroshareDirModel::preMods()
{
emit layoutAboutToBeChanged();
#if QT_VERSION < 0x050000
reset();
#else
@ -877,7 +877,7 @@ void RetroshareDirModel::preMods()
/* Callback from */
void RetroshareDirModel::postMods()
{
emit layoutAboutToBeChanged();
// emit layoutAboutToBeChanged();
#if QT_VERSION >= 0x040600
beginResetModel();
#endif
@ -890,7 +890,7 @@ void RetroshareDirModel::postMods()
// changePersistentIndexList(piList, empty);
/* Clear caches */
mCache.clear();
//mCache.clear();
#ifdef RDM_DEBUG
std::cerr << "RetroshareDirModel::postMods()" << std::endl;
@ -902,32 +902,36 @@ void RetroshareDirModel::postMods()
emit layoutChanged();
}
const DirDetailsVector *RetroshareDirModel::requestDirDetails(void *ref, bool remote) const
bool RetroshareDirModel::requestDirDetails(void *ref, bool remote,DirDetails& d) const
{
const QMap<void*, DirDetailsVector>::const_iterator it = mCache.constFind(ref);
if (it != mCache.constEnd()) {
/* Details found in cache */
return &it.value();
}
/* Get details from the lib */
DirDetailsVector details;
FileSearchFlags flags = (remote) ? RS_FILE_HINTS_REMOTE : RS_FILE_HINTS_LOCAL;
if (rsFiles->RequestDirDetails(ref, details, flags)) {
/* Convert std::list to std::vector for fast access with index */
std::list<DirStub>::const_iterator childIt;
for (childIt = details.children.begin(); childIt != details.children.end(); ++childIt) {
details.childrenVector.push_back(*childIt);
}
/* Add to cache, must cast to none const */
const QMap<void*, DirDetailsVector>::iterator it1 = ((QMap<void*, DirDetailsVector>*) &mCache)->insert(ref, details);
return &it1.value();
}
/* No details found */
return NULL;
return rsFiles->RequestDirDetails(ref, d, flags) ;
}
// const QMap<void*, DirDetailsVector>::const_iterator it = mCache.constFind(ref);
// if (it != mCache.constEnd()) {
// /* Details found in cache */
// return &it.value();
// }
//
//#warning this is terrible! A vector in a std::map will keep being copied and create a lot of CPU overload. Use a pointer instead.
// /* Get details from the lib */
//
// if (rsFiles->RequestDirDetails(ref, details, flags)) {
// /* Convert std::list to std::vector for fast access with index */
// std::list<DirStub>::const_iterator childIt;
// for (childIt = details.children.begin(); childIt != details.children.end(); ++childIt) {
// details.childrenVector.push_back(*childIt);
// }
//
// /* Add to cache, must cast to none const */
// const QMap<void*, DirDetailsVector>::iterator it1 = ((QMap<void*, DirDetailsVector>*) &mCache)->insert(ref, details);
// return &it1.value();
// }
//
// /* No details found */
// return NULL;
//}
void RetroshareDirModel::createCollectionFile(QWidget *parent, const QModelIndexList &list)
{
@ -1010,15 +1014,16 @@ void RetroshareDirModel::downloadDirectory(const DirDetails & dirDetails, int pr
if (!dwlDir.mkpath(cleanPath)) return;
for (it = dirDetails.children.begin(); it != dirDetails.children.end(); ++it)
for(uint32_t i=0;i<dirDetails.children.size();++i)
{
if (!it->ref) continue;
if (!dirDetails.children[i].ref) continue;
const DirDetails *subDirDetails = requestDirDetails(it->ref, true);
DirDetails subDirDetails ;
if (!subDirDetails) continue;
if(!requestDirDetails(dirDetails.children[i].ref, true,subDirDetails))
continue;
downloadDirectory(*subDirDetails, prefixLen);
downloadDirectory(subDirDetails, prefixLen);
}
}
}
@ -1035,12 +1040,12 @@ void RetroshareDirModel::getDirDetailsFromSelect (const QModelIndexList &list, s
{
void *ref = it -> internalPointer();
const DirDetails *details = requestDirDetails(ref, RemoteMode);
DirDetails details ;
if (!details)
if(!requestDirDetails(ref, RemoteMode,details))
continue;
dirVec.push_back(*details);
dirVec.push_back(details);
}
}
}
@ -1072,12 +1077,12 @@ void RetroshareDirModel::getFileInfoFromIndexList(const QModelIndexList& list, s
{
void *ref = it -> internalPointer();
const DirDetails *details = requestDirDetails(ref, RemoteMode);
DirDetails details;
if (!details)
if (!requestDirDetails(ref, RemoteMode,details))
continue;
if(details->type == DIR_TYPE_PERSON)
if(details.type == DIR_TYPE_PERSON)
continue ;
#ifdef RDM_DEBUG
@ -1090,10 +1095,10 @@ void RetroshareDirModel::getFileInfoFromIndexList(const QModelIndexList& list, s
// Note: for directories, the returned hash, is the peer id, so if we collect
// dirs, we need to be a bit more conservative for the
if(already_in.find(details->hash.toStdString()+details->name) == already_in.end())
if(already_in.find(details.hash.toStdString()+details.name) == already_in.end())
{
file_details.push_back(*details) ;
already_in.insert(details->hash.toStdString()+details->name) ;
file_details.push_back(details) ;
already_in.insert(details.hash.toStdString()+details.name) ;
}
}
#ifdef RDM_DEBUG
@ -1149,11 +1154,29 @@ void RetroshareDirModel::openSelected(const QModelIndexList &qmil)
#endif
}
void RetroshareDirModel::getFilePath(const QModelIndex& index, std::string& fullpath)
{
void *ref = index.sibling(index.row(),1).internalPointer();
DirDetails details ;
if (!requestDirDetails(ref, false,details) )
{
#ifdef RDM_DEBUG
std::cerr << "getFilePaths() Bad Request" << std::endl;
#endif
return;
}
fullpath = details.path + "/" + details.name;
}
void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list<std::string> &fullpaths)
{
#ifdef RDM_DEBUG
std::cerr << "RetroshareDirModel::getFilePaths()" << std::endl;
#endif
#warning make sure we atually output something here
if (RemoteMode)
{
#ifdef RDM_DEBUG
@ -1162,47 +1185,17 @@ void RetroshareDirModel::getFilePaths(const QModelIndexList &list, std::list<std
return;
}
/* translate */
QModelIndexList::const_iterator it;
for(it = list.begin(); it != list.end(); ++it)
for(QModelIndexList::const_iterator it = list.begin(); it != list.end(); ++it)
{
void *ref = it -> internalPointer();
std::string path ;
const DirDetails *details = requestDirDetails(ref, false);
if (!details)
{
getFilePath(*it,path) ;
#ifdef RDM_DEBUG
std::cerr << "getFilePaths() Bad Request" << std::endl;
std::cerr << "Constructed FilePath: " << path << std::endl;
#endif
continue;
}
if (details->type != DIR_TYPE_FILE)
{
#ifdef RDM_DEBUG
std::cerr << "getFilePaths() Not File" << std::endl;
#endif
continue; /* not file! */
}
#ifdef RDM_DEBUG
std::cerr << "::::::::::::File Details:::: " << std::endl;
std::cerr << "Name: " << details->name << std::endl;
std::cerr << "Hash: " << details->hash << std::endl;
std::cerr << "Size: " << details->count << std::endl;
std::cerr << "Path: " << details->path << std::endl;
#endif
std::string filepath = details->path + "/";
filepath += details->name;
#ifdef RDM_DEBUG
std::cerr << "Constructed FilePath: " << filepath << std::endl;
#endif
if (fullpaths.end() == std::find(fullpaths.begin(), fullpaths.end(), filepath))
{
fullpaths.push_back(filepath);
}
#warning TERRIBLE COST here. Use a std::set!
if (fullpaths.end() == std::find(fullpaths.begin(), fullpaths.end(), path))
fullpaths.push_back(path);
}
#ifdef RDM_DEBUG
std::cerr << "::::::::::::Done getFilePaths" << std::endl;
@ -1222,12 +1215,9 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con
{
void *ref = it -> internalPointer();
const DirDetails *details = requestDirDetails(ref, RemoteMode);
if (!details)
{
DirDetails details ;
if (!requestDirDetails(ref, RemoteMode,details))
continue;
}
#ifdef RDM_DEBUG
std::cerr << "::::::::::::FileDrag:::: " << std::endl;
@ -1237,7 +1227,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con
std::cerr << "Path: " << details->path << std::endl;
#endif
if (details->type != DIR_TYPE_FILE)
if (details.type != DIR_TYPE_FILE)
{
#ifdef RDM_DEBUG
std::cerr << "RetroshareDirModel::mimeData() Not File" << std::endl;
@ -1245,7 +1235,7 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con
continue; /* not file! */
}
if (drags.end() != (dit = drags.find(details->hash)))
if (drags.end() != (dit = drags.find(details.hash)))
{
#ifdef RDM_DEBUG
std::cerr << "RetroshareDirModel::mimeData() Duplicate" << std::endl;
@ -1253,9 +1243,9 @@ QMimeData * RetroshareDirModel::mimeData ( const QModelIndexList & indexes ) con
continue; /* duplicate */
}
drags[details->hash] = details->count;
drags[details.hash] = details.count;
QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details->name.c_str()), QString::fromStdString(details->hash.toStdString()), QString::number(details->count));
QString line = QString("%1/%2/%3/").arg(QString::fromUtf8(details.name.c_str()), QString::fromStdString(details.hash.toStdString()), QString::number(details.count));
if (RemoteMode)
{
@ -1315,14 +1305,15 @@ void FlatStyle_RDM::postMods()
{
if(visible())
{
_ref_entries.clear() ;
emit layoutAboutToBeChanged();
{
RS_STACK_MUTEX(_ref_mutex) ;
_ref_stack.clear() ;
_ref_stack.push_back(NULL) ; // init the stack with the topmost parent directory
std::cerr << "FlatStyle_RDM::postMods(): cleared ref entries" << std::endl;
_needs_update = false ;
updateRefs() ;
}
QTimer::singleShot(100,this,SLOT(updateRefs())) ;
}
else
_needs_update = true ;
@ -1338,9 +1329,17 @@ void FlatStyle_RDM::updateRefs()
RetroshareDirModel::preMods() ;
static const uint32_t MAX_REFS_PER_SECOND = 2000 ;
uint32_t nb_treated_refs = 0 ;
{
RS_STACK_MUTEX(_ref_mutex) ;
_ref_entries.clear() ;
std::cerr << "FlatStyle_RDM::postMods(): cleared ref entries" << std::endl;
while(!_ref_stack.empty())
{
void *ref = _ref_stack.back() ;
@ -1348,17 +1347,18 @@ void FlatStyle_RDM::updateRefs()
std::cerr << "FlatStyle_RDM::postMods(): poped ref " << ref << std::endl;
#endif
_ref_stack.pop_back() ;
const DirDetails *details = requestDirDetails(ref, RemoteMode) ;
if (details)
DirDetails details ;
if (requestDirDetails(ref, RemoteMode,details))
{
if(details->type == DIR_TYPE_FILE) // only push files, not directories nor persons.
_ref_entries.push_back(std::pair<void*,QString>(ref,computeDirectoryPath(*details)));
if(details.type == DIR_TYPE_FILE) // only push files, not directories nor persons.
_ref_entries.push_back(ref) ;
#ifdef RDM_DEBUG
std::cerr << "FlatStyle_RDM::postMods(): adding ref " << ref << std::endl;
#endif
for(std::list<DirStub>::const_iterator it = details->children.begin(); it != details->children.end(); ++it)
_ref_stack.push_back(it->ref) ;
for(uint32_t i=0;i<details.children.size();++i)
_ref_stack.push_back(details.children[i].ref) ;
}
if(++nb_treated_refs > MAX_REFS_PER_SECOND) // we've done enough, let's give back hand to
{ // the user and setup a timer to finish the job later.
@ -1375,6 +1375,7 @@ void FlatStyle_RDM::updateRefs()
if(_ref_stack.empty())
_needs_update = false ;
}
RetroshareDirModel::postMods() ;
}

13
retroshare-gui/src/gui/RemoteDirModel.h
View File

@ -72,11 +72,14 @@ class RetroshareDirModel : public QAbstractItemModel
void getFileInfoFromIndexList(const QModelIndexList& list, std::list<DirDetails>& files_info) ;
void openSelected(const QModelIndexList &list);
void getFilePaths(const QModelIndexList &list, std::list<std::string> &fullpaths);
void getFilePath(const QModelIndex& index, std::string& fullpath);
void changeAgeIndicator(uint32_t indicator) { ageIndicator = indicator; }
const DirDetailsVector *requestDirDetails(void *ref, bool remote) const;
bool requestDirDetails(void *ref, bool remote,DirDetails& d) const;
virtual void update() {}
virtual void updateRef(const QModelIndex&) const =0;
public:
virtual QMimeData * mimeData ( const QModelIndexList & indexes ) const;
virtual QStringList mimeTypes () const;
@ -166,6 +169,8 @@ class TreeStyle_RDM: public RetroshareDirModel
virtual ~TreeStyle_RDM() ;
protected:
virtual void updateRef(const QModelIndex&) const ;
/* These are all overloaded Virtual Functions */
virtual int rowCount(const QModelIndex &parent = QModelIndex()) const;
virtual int columnCount(const QModelIndex &parent = QModelIndex()) const;
@ -189,7 +194,7 @@ class FlatStyle_RDM: public RetroshareDirModel
public:
FlatStyle_RDM(bool mode)
: RetroshareDirModel(mode)
: RetroshareDirModel(mode), _ref_mutex("Flat file list")
{
_needs_update = true ;
}
@ -202,6 +207,7 @@ class FlatStyle_RDM: public RetroshareDirModel
void updateRefs() ;
protected:
virtual void updateRef(const QModelIndex&) const {}
virtual void postMods();
virtual int rowCount(const QModelIndex &parent = QModelIndex()) const;
@ -217,7 +223,8 @@ class FlatStyle_RDM: public RetroshareDirModel
QString computeDirectoryPath(const DirDetails& details) const ;
std::vector<std::pair<void *,QString> > _ref_entries ;// used to store the refs to display
mutable RsMutex _ref_mutex ;
std::vector<void *> _ref_entries ;// used to store the refs to display
std::vector<void *> _ref_stack ; // used to store the refs to update
bool _needs_update ;
};

8
retroshare-gui/src/gui/ShareManager.cpp
View File

@ -148,7 +148,7 @@ void ShareManager::load()
GroupFlagsWidget *widget = new GroupFlagsWidget(NULL,(*it).shareflags);
listWidget->setRowHeight(row, 32);
listWidget->setRowHeight(row, 32 * QFontMetricsF(font()).height()/14.0);
listWidget->setCellWidget(row, COLUMN_SHARE_FLAGS, widget);
listWidget->setItem(row, COLUMN_GROUPS, new QTableWidgetItem()) ;
@ -157,7 +157,7 @@ void ShareManager::load()
//connect(widget,SIGNAL(flagsChanged(FileStorageFlags)),this,SLOT(updateFlags())) ;
}
listWidget->setColumnWidth(COLUMN_SHARE_FLAGS,132) ;
listWidget->setColumnWidth(COLUMN_SHARE_FLAGS,132 * QFontMetricsF(font()).height()/14.0) ;
//ui.incomingDir->setText(QString::fromStdString(rsFiles->getDownloadDirectory()));
@ -282,6 +282,7 @@ void ShareManager::editShareDirectory()
ShareDialog sharedlg (it->filename, this);
sharedlg.setWindowTitle(tr("Edit Shared Folder"));
sharedlg.exec();
load();
break;
}
}
@ -318,6 +319,7 @@ void ShareManager::showShareDialog()
{
ShareDialog sharedlg ("", this);
sharedlg.exec();
load();
}
void ShareManager::shareddirListCurrentCellChanged(int currentRow, int currentColumn, int previousRow, int previousColumn)
@ -387,4 +389,6 @@ void ShareManager::dropEvent(QDropEvent *event)
event->setDropAction(Qt::CopyAction);
event->accept();
load();
}

129
retroshare-gui/src/gui/SharedFilesDialog.cpp
View File

@ -19,6 +19,8 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
#include <set>
#include <QString>
#include <QTreeView>
#include <QClipboard>
@ -83,6 +85,8 @@
//
#define DONT_USE_SEARCH_IN_TREE_VIEW 1
//#define DEBUG_SHARED_FILES_DIALOG 1
const QString Image_AddNewAssotiationForFile = ":/images/kcmsystem24.png";
class SFDSortFilterProxyModel : public QSortFilterProxyModel
@ -134,19 +138,22 @@ SharedFilesDialog::SharedFilesDialog(RetroshareDirModel *_tree_model,RetroshareD
flat_model = _flat_model ;
tree_proxyModel = new SFDSortFilterProxyModel(tree_model, this);
tree_proxyModel->setDynamicSortFilter(true);
tree_proxyModel->setSourceModel(tree_model);
tree_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive);
tree_proxyModel->setSortRole(RetroshareDirModel::SortRole);
tree_proxyModel->sort(COLUMN_NAME);
flat_proxyModel = new SFDSortFilterProxyModel(flat_model, this);
flat_proxyModel->setDynamicSortFilter(true);
flat_proxyModel->setSourceModel(flat_model);
flat_proxyModel->setSortCaseSensitivity(Qt::CaseInsensitive);
flat_proxyModel->setSortRole(RetroshareDirModel::SortRole);
flat_proxyModel->sort(COLUMN_NAME);
// Mr.Alice: I removed this because it causes a crash for some obscur reason. Apparently when the model is changed, the proxy model cannot
// deal with the change by itself. Should I call something specific? I've no idea. Removing this does not seem to cause any harm either.
//tree_proxyModel->setDynamicSortFilter(true);
//flat_proxyModel->setDynamicSortFilter(true);
connect(ui.filterClearButton, SIGNAL(clicked()), this, SLOT(clearFilter()));
connect(ui.filterStartButton, SIGNAL(clicked()), this, SLOT(startFilter()));
connect(ui.filterPatternLineEdit, SIGNAL(returnPressed()), this, SLOT(startFilter()));
@ -224,6 +231,7 @@ RemoteSharedFilesDialog::RemoteSharedFilesDialog(QWidget *parent)
ui.checkButton->hide() ;
connect(ui.downloadButton, SIGNAL(clicked()), this, SLOT(downloadRemoteSelected()));
connect(ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), this, SLOT( expanded(const QModelIndex & ) ) );
// load settings
processSettings(true);
@ -248,8 +256,13 @@ void SharedFilesDialog::showEvent(QShowEvent *)
{
if(model!=NULL)
{
std::set<std::string> expanded_indexes ;
saveExpandedPaths(expanded_indexes);
model->setVisible(true) ;
model->update() ;
restoreExpandedPaths(expanded_indexes);
}
}
RemoteSharedFilesDialog::~RemoteSharedFilesDialog()
@ -346,23 +359,29 @@ void SharedFilesDialog::changeCurrentViewModel(int viewTypeIndex)
showProperColumns() ;
std::set<std::string> expanded_indexes ;
saveExpandedPaths(expanded_indexes);
if(isVisible())
{
model->setVisible(true) ;
model->update() ;
}
//connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), model, SLOT( collapsed(const QModelIndex & ) ) );
//connect( ui.dirTreeView, SIGNAL( expanded(const QModelIndex & ) ), model, SLOT( expanded(const QModelIndex & ) ) );
// connect( ui.dirTreeView, SIGNAL( collapsed(const QModelIndex & ) ), this, SLOT( collapsed(const QModelIndex & ) ) );
ui.dirTreeView->setModel(proxyModel);
ui.dirTreeView->update();
restoreExpandedPaths(expanded_indexes);
QHeaderView * header = ui.dirTreeView->header () ;
QHeaderView_setSectionResizeModeColumn(header, COLUMN_NAME, QHeaderView::Interactive);
ui.dirTreeView->header()->headerDataChanged(Qt::Horizontal, COLUMN_NAME, COLUMN_DIR) ;
// recursRestoreExpandedItems(ui.dirTreeView->rootIndex(),expanded_indexes);
FilterItems();
}
@ -501,6 +520,15 @@ QModelIndexList SharedFilesDialog::getSelected()
return proxyList ;
}
void RemoteSharedFilesDialog::expanded(const QModelIndex& indx)
{
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "Expanding at " << indx.row() << " and " << indx.column() << " ref=" << indx.internalPointer() << ", pointer at 1: " << proxyModel->mapToSource(indx).internalPointer() << std::endl;
#endif
model->updateRef(proxyModel->mapToSource(indx)) ;
}
void RemoteSharedFilesDialog::downloadRemoteSelected()
{
/* call back to the model (which does all the interfacing? */
@ -544,9 +572,9 @@ void SharedFilesDialog::copyLink (const QModelIndexList& lst, bool remote)
if (details.type == DIR_TYPE_DIR)
{
for (std::list<DirStub>::const_iterator cit = details.children.begin();cit != details.children.end(); ++cit)
for(uint32_t j=0;j<details.children.size();++j)
{
const DirStub& dirStub = *cit;
const DirStub& dirStub = details.children[j];
DirDetails details;
FileSearchFlags flags = remote?RS_FILE_HINTS_REMOTE:RS_FILE_HINTS_LOCAL ;
@ -823,20 +851,109 @@ void SharedFilesDialog::preModDirectories(bool local)
flat_model->preMods();
}
void SharedFilesDialog::saveExpandedPaths(std::set<std::string>& expanded_indexes)
{
if(ui.dirTreeView->model() == NULL)
return ;
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "Saving expanded items. " << std::endl;
#endif
for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row)
{
std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString();
recursSaveExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes);
}
}
void SharedFilesDialog::restoreExpandedPaths(const std::set<std::string>& expanded_indexes)
{
if(ui.dirTreeView->model() == NULL)
return ;
// we need to disable this, because the signal will trigger unnecessary update at the friend.
ui.dirTreeView->blockSignals(true) ;
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "Restoring expanded items. " << std::endl;
#endif
for(int row = 0; row < ui.dirTreeView->model()->rowCount(); ++row)
{
std::string path = ui.dirTreeView->model()->index(row,0).data(Qt::DisplayRole).toString().toStdString();
recursRestoreExpandedItems(ui.dirTreeView->model()->index(row,0),path,expanded_indexes);
}
ui.dirTreeView->blockSignals(false) ;
}
void SharedFilesDialog::recursSaveExpandedItems(const QModelIndex& index,const std::string& path,std::set<std::string>& exp)
{
std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString();
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl;
#endif
if(ui.dirTreeView->isExpanded(index))
{
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "Index " << local_path << " is expanded." << std::endl;
#endif
if(index.isValid())
exp.insert(local_path) ;
for(int row=0;row<ui.dirTreeView->model()->rowCount(index);++row)
recursSaveExpandedItems(index.child(row,0),local_path,exp) ;
}
#ifdef DEBUG_SHARED_FILES_DIALOG
else
std::cerr << "Index is not expanded." << std::endl;
#endif
}
void SharedFilesDialog::recursRestoreExpandedItems(const QModelIndex& index, const std::string &path, const std::set<std::string>& exp)
{
std::string local_path = path+"/"+index.data(Qt::DisplayRole).toString().toStdString();
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "at index " << index.row() << ". data[1]=" << local_path << std::endl;
#endif
if(exp.find(local_path) != exp.end())
{
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "re expanding index " << local_path << std::endl;
#endif
ui.dirTreeView->setExpanded(index,true) ;
for(int row=0;row<ui.dirTreeView->model()->rowCount(index);++row)
recursRestoreExpandedItems(index.child(row,0),local_path,exp) ;
}
}
void SharedFilesDialog::postModDirectories(bool local)
{
if (isRemote() == local) {
return;
}
std::set<std::string> expanded_indexes;
saveExpandedPaths(expanded_indexes) ;
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "Saving expanded items. " << expanded_indexes.size() << " items found" << std::endl;
#endif
/* Notify both models, only one is visible */
tree_model->postMods();
flat_model->postMods();
ui.dirTreeView->update() ;
restoreExpandedPaths(expanded_indexes) ;
if (ui.filterPatternLineEdit->text().isEmpty() == false)
FilterItems();
#ifdef DEBUG_SHARED_FILES_DIALOG
std::cerr << "****** updated directories! ******" << std::endl;
#endif
QCoreApplication::flush();
}

8
retroshare-gui/src/gui/SharedFilesDialog.h
View File

@ -22,6 +22,7 @@
#ifndef _SHAREDFILESDIALOG_H
#define _SHAREDFILESDIALOG_H
#include <set>
#include "RsAutoUpdatePage.h"
#include "ui_SharedFilesDialog.h"
@ -51,7 +52,6 @@ protected slots:
virtual void spawnCustomPopupMenu(QPoint point) = 0;
private slots:
/* For handling the model updates */
void preModDirectories(bool local) ;
void postModDirectories(bool local) ;
@ -94,6 +94,11 @@ protected:
Ui::SharedFilesDialog ui;
virtual void processSettings(bool bLoad) = 0;
void recursRestoreExpandedItems(const QModelIndex& index,const std::string& path,const std::set<std::string>& exp);
void recursSaveExpandedItems(const QModelIndex& index, const std::string &path, std::set<std::string> &exp);
void saveExpandedPaths(std::set<std::string>& paths) ;
void restoreExpandedPaths(const std::set<std::string>& paths) ;
protected:
//now context menu are created again every time theu are called ( in some
//slots.. Maybe it's not good...
@ -191,6 +196,7 @@ class RemoteSharedFilesDialog : public SharedFilesDialog
private slots:
void downloadRemoteSelected();
void expanded(const QModelIndex& indx);
};
#endif

6
retroshare-gui/src/gui/advsearch/advancedsearchdialog.cpp
View File

@ -119,9 +119,9 @@ void AdvancedSearchDialog::prepareSearch()
}
Expression * AdvancedSearchDialog::getRsExpr()
RsRegularExpression::Expression * AdvancedSearchDialog::getRsExpr()
{
Expression * wholeExpression;
RsRegularExpression::Expression * wholeExpression;
// process the special case: first expression
wholeExpression = expressions->at(0)->getRsExpression();
@ -131,7 +131,7 @@ Expression * AdvancedSearchDialog::getRsExpr()
for (int i = 1; i < expressions->size(); ++i) {
// extract the expression information and compound it with the
// first expression
wholeExpression = new CompoundExpression(expressions->at(i)->getOperator(),
wholeExpression = new RsRegularExpression::CompoundExpression(expressions->at(i)->getOperator(),
wholeExpression,
expressions->at(i)->getRsExpression());
}

4
retroshare-gui/src/gui/advsearch/advancedsearchdialog.h
View File

@ -36,10 +36,10 @@ class AdvancedSearchDialog : public QDialog, public Ui::AdvancedSearchDialog
public:
AdvancedSearchDialog(QWidget * parent = 0 );
Expression * getRsExpr();
RsRegularExpression::Expression * getRsExpr();
QString getSearchAsString();
signals:
void search(Expression*);
void search(RsRegularExpression::Expression*);
private slots:
void deleteExpression(ExpressionWidget*);

30
retroshare-gui/src/gui/advsearch/expressionwidget.cpp
View File

@ -130,7 +130,7 @@ void ExpressionWidget::deleteExpression()
emit signalDelete(this);
}
LogicalOperator ExpressionWidget::getOperator()
RsRegularExpression::LogicalOperator ExpressionWidget::getOperator()
{
return exprOpElem->getLogicalOperator();
}
@ -145,9 +145,9 @@ static int checkedConversion(uint64_t s)
return (int)s ;
}
Expression* ExpressionWidget::getRsExpression()
RsRegularExpression::Expression* ExpressionWidget::getRsExpression()
{
Expression * expr = NULL;
RsRegularExpression::Expression * expr = NULL;
std::list<std::string> wordList;
uint64_t lowVal = 0;
@ -174,54 +174,54 @@ Expression* ExpressionWidget::getRsExpression()
switch (searchType)
{
case NameSearch:
expr = new NameExpression(exprCondElem->getStringOperator(),
expr = new RsRegularExpression::NameExpression(exprCondElem->getStringOperator(),
wordList,
exprParamElem->ignoreCase());
break;
case PathSearch:
expr = new PathExpression(exprCondElem->getStringOperator(),
expr = new RsRegularExpression::PathExpression(exprCondElem->getStringOperator(),
wordList,
exprParamElem->ignoreCase());
break;
case ExtSearch:
expr = new ExtExpression(exprCondElem->getStringOperator(),
expr = new RsRegularExpression::ExtExpression(exprCondElem->getStringOperator(),
wordList,
exprParamElem->ignoreCase());
break;
case HashSearch:
expr = new HashExpression(exprCondElem->getStringOperator(),
expr = new RsRegularExpression::HashExpression(exprCondElem->getStringOperator(),
wordList);
break;
case DateSearch:
if (inRangedConfig) {
expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal));
expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal));
} else {
expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue()));
expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue()));
}
break;
case PopSearch:
if (inRangedConfig) {
expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal));
expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(lowVal), checkedConversion(highVal));
} else {
expr = new DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue()));
expr = new RsRegularExpression::DateExpression(exprCondElem->getRelOperator(), checkedConversion(exprParamElem->getIntValue()));
}
break;
case SizeSearch:
if (inRangedConfig)
{
if(lowVal >= (uint64_t)(1024*1024*1024) || highVal >= (uint64_t)(1024*1024*1024))
expr = new SizeExpressionMB(exprCondElem->getRelOperator(), (int)(lowVal / (1024*1024)), (int)(highVal / (1024*1024)));
expr = new RsRegularExpression::SizeExpressionMB(exprCondElem->getRelOperator(), (int)(lowVal / (1024*1024)), (int)(highVal / (1024*1024)));
else
expr = new SizeExpression(exprCondElem->getRelOperator(), lowVal, highVal);
expr = new RsRegularExpression::SizeExpression(exprCondElem->getRelOperator(), lowVal, highVal);
}
else
{
uint64_t s = exprParamElem->getIntValue() ;
if(s >= (uint64_t)(1024*1024*1024))
expr = new SizeExpressionMB(exprCondElem->getRelOperator(), (int)(s/(1024*1024))) ;
expr = new RsRegularExpression::SizeExpressionMB(exprCondElem->getRelOperator(), (int)(s/(1024*1024))) ;
else
expr = new SizeExpression(exprCondElem->getRelOperator(), (int)s) ;
expr = new RsRegularExpression::SizeExpression(exprCondElem->getRelOperator(), (int)s) ;
}
break;
};

4
retroshare-gui/src/gui/advsearch/expressionwidget.h
View File

@ -45,11 +45,11 @@ public:
/** delivers the expression represented by this widget
the operator to join this expression with any previous
expressions is provided by the getOperator method */
Expression* getRsExpression();
RsRegularExpression::Expression* getRsExpression();
/** supplies the operator to be used when joining this expression
to the whole query */
LogicalOperator getOperator();
RsRegularExpression::LogicalOperator getOperator();
QString toString();

36
retroshare-gui/src/gui/advsearch/guiexprelement.cpp
View File

@ -62,9 +62,9 @@ QStringList GuiExprElement::relOptionsList;
QMap<int, ExprSearchType> GuiExprElement::TermsIndexMap;
QMap<int, LogicalOperator> GuiExprElement::logicalOpIndexMap;
QMap<int, StringOperator> GuiExprElement::strConditionIndexMap;
QMap<int, RelOperator> GuiExprElement::relConditionIndexMap;
QMap<int, RsRegularExpression::LogicalOperator> GuiExprElement::logicalOpIndexMap;
QMap<int, RsRegularExpression::StringOperator> GuiExprElement::strConditionIndexMap;
QMap<int, RsRegularExpression::RelOperator> GuiExprElement::relConditionIndexMap;
QMap<int, QString> GuiExprElement::logicalOpStrMap;
QMap<int, QString> GuiExprElement::termsStrMap;
@ -141,9 +141,9 @@ void GuiExprElement::initialiseOptionsLists()
GuiExprElement::relOptionsList.append(RANGE);
// now the maps
GuiExprElement::logicalOpIndexMap[GuiExprElement::AND_INDEX] = AndOp;
GuiExprElement::logicalOpIndexMap[GuiExprElement::OR_INDEX] = OrOp;
GuiExprElement::logicalOpIndexMap[GuiExprElement::XOR_INDEX] = XorOp;
GuiExprElement::logicalOpIndexMap[GuiExprElement::AND_INDEX] = RsRegularExpression::AndOp;
GuiExprElement::logicalOpIndexMap[GuiExprElement::OR_INDEX] = RsRegularExpression::OrOp;
GuiExprElement::logicalOpIndexMap[GuiExprElement::XOR_INDEX] = RsRegularExpression::XorOp;
GuiExprElement::TermsIndexMap[GuiExprElement::NAME_INDEX] = NameSearch;
GuiExprElement::TermsIndexMap[GuiExprElement::PATH_INDEX] = PathSearch;
@ -153,9 +153,9 @@ void GuiExprElement::initialiseOptionsLists()
GuiExprElement::TermsIndexMap[GuiExprElement::SIZE_INDEX] = SizeSearch;
GuiExprElement::TermsIndexMap[GuiExprElement::POP_INDEX] = PopSearch;
GuiExprElement::strConditionIndexMap[GuiExprElement::CONTAINS_INDEX] = ContainsAnyStrings;
GuiExprElement::strConditionIndexMap[GuiExprElement::CONTALL_INDEX] = ContainsAllStrings;
GuiExprElement::strConditionIndexMap[GuiExprElement::IS_INDEX] = EqualsString;
GuiExprElement::strConditionIndexMap[GuiExprElement::CONTAINS_INDEX] = RsRegularExpression::ContainsAnyStrings;
GuiExprElement::strConditionIndexMap[GuiExprElement::CONTALL_INDEX] = RsRegularExpression::ContainsAllStrings;
GuiExprElement::strConditionIndexMap[GuiExprElement::IS_INDEX] = RsRegularExpression::EqualsString;
/* W A R N I N G !!!!
the cb elements correspond to their inverse rel op counterparts in rsexpr.h due to the nature of
@ -166,12 +166,12 @@ void GuiExprElement::initialiseOptionsLists()
files where the condition is greater than the file size i.e. files whose size is less than or equal to the condition
Therefore we invert the mapping of rel conditions here to match the behaviour of the impl.
*/
GuiExprElement::relConditionIndexMap[GuiExprElement::LT_INDEX] = GreaterEquals;
GuiExprElement::relConditionIndexMap[GuiExprElement::LTE_INDEX] = Greater;
GuiExprElement::relConditionIndexMap[GuiExprElement::EQUAL_INDEX] = Equals;
GuiExprElement::relConditionIndexMap[GuiExprElement::GTE_INDEX] = Smaller;
GuiExprElement::relConditionIndexMap[GuiExprElement::GT_INDEX] = SmallerEquals;
GuiExprElement::relConditionIndexMap[GuiExprElement::RANGE_INDEX] = InRange;
GuiExprElement::relConditionIndexMap[GuiExprElement::LT_INDEX] = RsRegularExpression::GreaterEquals;
GuiExprElement::relConditionIndexMap[GuiExprElement::LTE_INDEX] = RsRegularExpression::Greater;
GuiExprElement::relConditionIndexMap[GuiExprElement::EQUAL_INDEX] = RsRegularExpression::Equals;
GuiExprElement::relConditionIndexMap[GuiExprElement::GTE_INDEX] = RsRegularExpression::Smaller;
GuiExprElement::relConditionIndexMap[GuiExprElement::GT_INDEX] = RsRegularExpression::SmallerEquals;
GuiExprElement::relConditionIndexMap[GuiExprElement::RANGE_INDEX] = RsRegularExpression::InRange;
// the string to index map
GuiExprElement::termsStrMap[GuiExprElement::NAME_INDEX] = NAME;
@ -260,7 +260,7 @@ QString ExprOpElement::toString()
}
LogicalOperator ExprOpElement::getLogicalOperator()
RsRegularExpression::LogicalOperator ExprOpElement::getLogicalOperator()
{
return GuiExprElement::logicalOpIndexMap[cb->currentIndex()];
}
@ -313,12 +313,12 @@ QString ExprConditionElement::toString()
return GuiExprElement::relConditionStrMap[cb->currentIndex()];
}
RelOperator ExprConditionElement::getRelOperator()
RsRegularExpression::RelOperator ExprConditionElement::getRelOperator()
{
return GuiExprElement::relConditionIndexMap[cb->currentIndex()];
}
StringOperator ExprConditionElement::getStringOperator()
RsRegularExpression::StringOperator ExprConditionElement::getStringOperator()
{
return GuiExprElement::strConditionIndexMap[cb->currentIndex()];
}

16
retroshare-gui/src/gui/advsearch/guiexprelement.h
View File

@ -114,9 +114,9 @@ protected:
static QStringList relOptionsList;
// provides a mapping of condition operators to RSExpr reloperators
static QMap<int, LogicalOperator> logicalOpIndexMap;
static QMap<int, StringOperator> strConditionIndexMap;
static QMap<int, RelOperator> relConditionIndexMap;
static QMap<int, RsRegularExpression::LogicalOperator> logicalOpIndexMap;
static QMap<int, RsRegularExpression::StringOperator> strConditionIndexMap;
static QMap<int, RsRegularExpression::RelOperator> relConditionIndexMap;
// provides a mapping of indexes to translatable strings
static QMap<int, QString> logicalOpStrMap;
@ -134,7 +134,7 @@ class ExprOpElement : public GuiExprElement
public:
ExprOpElement(QWidget * parent = 0);
LogicalOperator getLogicalOperator();
RsRegularExpression::LogicalOperator getLogicalOperator();
QString toString();
private:
QComboBox * cb;
@ -148,8 +148,8 @@ class ExprTermsElement : public GuiExprElement
public:
ExprTermsElement(QWidget * parent = 0);
int getTermsIndex();
RelOperator getRelOperator();
StringOperator getStringOperator();
RsRegularExpression::RelOperator getRelOperator();
RsRegularExpression::StringOperator getStringOperator();
void set(int i) {cb->setCurrentIndex(i);}
QString toString();
@ -167,8 +167,8 @@ class ExprConditionElement : public GuiExprElement
public:
ExprConditionElement(ExprSearchType, QWidget * parent = 0);
RelOperator getRelOperator();
StringOperator getStringOperator();
RsRegularExpression::RelOperator getRelOperator();
RsRegularExpression::StringOperator getStringOperator();
void adjustForSearchType(ExprSearchType);
void set(int i) {cb->setCurrentIndex(i);}
QString toString();

6
retroshare-gui/src/gui/common/GroupFlagsWidget.cpp
View File

@ -35,8 +35,8 @@ GroupFlagsWidget::GroupFlagsWidget(QWidget *parent,FileStorageFlags flags)
{
_layout = new QHBoxLayout(this) ;
setMinimumSize(128,32) ;
setMaximumSize(128,32) ;
setMinimumSize(128 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0) ;
setMaximumSize(128 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0) ;
setSizePolicy(QSizePolicy::Fixed,QSizePolicy::Fixed);
_icons[INDEX_GROUP_BROWSABLE] = new QIcon(FLAGS_GROUP_BROWSABLE_ICON) ;
@ -58,7 +58,7 @@ GroupFlagsWidget::GroupFlagsWidget(QWidget *parent,FileStorageFlags flags)
_buttons[i] = new QPushButton(this) ;
_buttons[i]->setCheckable(true) ;
_buttons[i]->setChecked(flags & _flags[i]) ;
_buttons[i]->setIconSize(QSize(32,32));
_buttons[i]->setIconSize(QSize(32 * QFontMetricsF(font()).height()/14.0,32 * QFontMetricsF(font()).height()/14.0));
update_button_state(_buttons[i]->isChecked(),i) ;
_layout->addWidget(_buttons[i]) ;
}

6
retroshare-gui/src/gui/common/RsCollectionFile.cpp
View File

@ -157,15 +157,15 @@ void RsCollectionFile::recursAddElements(QDomDocument& doc,const DirDetails& det
d.setAttribute(QString("name"),QString::fromUtf8(details.name.c_str())) ;
for (std::list<DirStub>::const_iterator it = details.children.begin(); it != details.children.end(); ++it)
for(uint32_t i=0;i<details.children.size();++i)
{
if (!it->ref)
if (!details.children[i].ref)
continue;
DirDetails subDirDetails;
FileSearchFlags flags = RS_FILE_HINTS_LOCAL;
if (!rsFiles->RequestDirDetails(it->ref, subDirDetails, flags))
if (!rsFiles->RequestDirDetails(details.children[i].ref, subDirDetails, flags))
continue;
recursAddElements(doc,subDirDetails,d) ;

49
retroshare-gui/src/gui/settings/DirectoriesPage.cpp
View File

@ -36,18 +36,9 @@ DirectoriesPage::DirectoriesPage(QWidget * parent, Qt::WindowFlags flags)
connect(ui.incomingButton, SIGNAL(clicked( bool ) ), this , SLOT( setIncomingDirectory() ) );
connect(ui.partialButton, SIGNAL(clicked( bool ) ), this , SLOT( setPartialsDirectory() ) );
connect(ui.editShareButton, SIGNAL(clicked()), this, SLOT(editDirectories()));
connect(ui.cleanHashCachePB, SIGNAL(clicked()), this, SLOT(clearHashCache()));
connect(ui.rememberHashesCB, SIGNAL(clicked(bool)), this, SLOT(clickedRememberHashes(bool)));
connect(ui.rememberHashesCB, SIGNAL(clicked(bool)), this, SLOT(toggleRememberHashes()));
connect(ui.autoCheckDirectories_CB, SIGNAL(clicked(bool)), this, SLOT(toggleAutoCheckDirectories(bool)));
}
void DirectoriesPage::clearHashCache()
{
if(QMessageBox::question(this, tr("Cache cleaning confirmation"), tr("This will forget any former hash of non shared files. Do you confirm ?"), QMessageBox::Yes | QMessageBox::No, QMessageBox::No) == QMessageBox::Yes)
rsFiles->clearHashCache() ;
}
void DirectoriesPage::toggleAutoCheckDirectories(bool b)
{
ui.autoCheckDirectoriesDelay_SB->setEnabled(b);
@ -58,28 +49,9 @@ void DirectoriesPage::editDirectories()
ShareManager::showYourself() ;
}
void DirectoriesPage::clickedRememberHashes(bool b)
{
if (!b) {
if (QMessageBox::question(this,tr("Cache cleaning confirmation"), tr("This will forget any former hash of non shared files. Do you confirm ?"), QMessageBox::Yes | QMessageBox::No, QMessageBox::No) == QMessageBox::No) {
ui.rememberHashesCB->setChecked(true);
}
}
}
void DirectoriesPage::toggleRememberHashes()
{
bool b = ui.rememberHashesCB->isChecked();
ui.rememberHashesSB->setEnabled(b);
ui.cleanHashCachePB->setEnabled(b);
}
/** Saves the changes on this page */
bool DirectoriesPage::save(QString &/*errmsg*/)
{
rsFiles->setRememberHashFilesDuration(ui.rememberHashesSB->value());
rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value());
std::string dir = ui.incomingDir->text().toUtf8().constData();
if (!dir.empty())
{
@ -92,18 +64,8 @@ bool DirectoriesPage::save(QString &/*errmsg*/)
rsFiles->setPartialsDirectory(dir);
}
if (ui.rememberHashesCB->isChecked()) {
rsFiles->setRememberHashFiles(true);
} else {
rsFiles->setRememberHashFiles(false);
rsFiles->clearHashCache() ;
}
if (ui.autoCheckDirectories_CB->isChecked()) {
rsFiles->setWatchEnabled(ui.autoCheckDirectories_CB->isChecked()) ;
rsFiles->setWatchPeriod(ui.autoCheckDirectoriesDelay_SB->value());
} else {
rsFiles->setWatchPeriod(-ui.autoCheckDirectoriesDelay_SB->value());
}
rsFiles->shareDownloadDirectory(ui.shareDownloadDirectoryCB->isChecked());
@ -115,14 +77,9 @@ void DirectoriesPage::load()
{
ui.shareDownloadDirectoryCB->setChecked(rsFiles->getShareDownloadDirectory());
ui.rememberHashesSB->setValue(rsFiles->rememberHashFilesDuration());
ui.rememberHashesCB->setChecked(rsFiles->rememberHashFiles());
toggleRememberHashes();
int u = rsFiles->watchPeriod() ;
ui.autoCheckDirectoriesDelay_SB->setValue(abs(u)) ;
ui.autoCheckDirectories_CB->setChecked(u>0) ;
ui.autoCheckDirectoriesDelay_SB->setEnabled(u>0) ;
ui.autoCheckDirectoriesDelay_SB->setValue(u) ;
ui.autoCheckDirectories_CB->setChecked(rsFiles->watchEnabled()) ; ;
ui.incomingDir->setText(QString::fromUtf8(rsFiles->getDownloadDirectory().c_str()));
ui.partialsDir->setText(QString::fromUtf8(rsFiles->getPartialsDirectory().c_str()));

3
retroshare-gui/src/gui/settings/DirectoriesPage.h
View File

@ -45,9 +45,6 @@ private slots:
void editDirectories() ;
void setIncomingDirectory();
void setPartialsDirectory();
void clearHashCache();
void clickedRememberHashes(bool);
void toggleRememberHashes();
void toggleAutoCheckDirectories(bool);
private:

50
retroshare-gui/src/gui/settings/DirectoriesPage.ui
View File

@ -6,7 +6,7 @@
<rect>
<x>0</x>
<y>0</y>
<width>485</width>
<width>895</width>
<height>549</height>
</rect>
</property>
@ -144,50 +144,7 @@
</layout>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QCheckBox" name="rememberHashesCB">
<property name="toolTip">
<string>Remember file hashes even if not shared.
This might be useful if you're sharing an
external HD, to avoid re-hashing files when
you plug it in.</string>
</property>
<property name="text">
<string>Remember hashed files for </string>
</property>
<property name="checked">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<widget class="QSpinBox" name="rememberHashesSB">
<property name="suffix">
<string> days</string>
</property>
<property name="minimum">
<number>1</number>
</property>
<property name="maximum">
<number>365</number>
</property>
<property name="value">
<number>10</number>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="cleanHashCachePB">
<property name="toolTip">
<string>Forget any hashed file that is not anymore shared.</string>
</property>
<property name="text">
<string>Clean Hash Cache</string>
</property>
</widget>
</item>
</layout>
<layout class="QHBoxLayout" name="horizontalLayout"/>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_3">
@ -240,9 +197,6 @@ you plug it in.</string>
<tabstops>
<tabstop>shareDownloadDirectoryCB</tabstop>
<tabstop>editShareButton</tabstop>
<tabstop>rememberHashesCB</tabstop>
<tabstop>rememberHashesSB</tabstop>
<tabstop>cleanHashCachePB</tabstop>
<tabstop>autoCheckDirectories_CB</tabstop>
<tabstop>autoCheckDirectoriesDelay_SB</tabstop>
<tabstop>incomingDir</tabstop>

10
retroshare-gui/src/gui/settings/TransferPage.cpp
View File

@ -36,7 +36,6 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags)
ui.setupUi(this);
ui._queueSize_SB->setValue(rsFiles->getQueueSize()) ;
ui._minPrioritized_SB->setValue(rsFiles->getMinPrioritizedTransfers()) ;
switch(rsFiles->defaultChunkStrategy())
{
@ -48,7 +47,6 @@ TransferPage::TransferPage(QWidget * parent, Qt::WindowFlags flags)
ui._diskSpaceLimit_SB->setValue(rsFiles->freeDiskSpaceLimit()) ;
QObject::connect(ui._queueSize_SB,SIGNAL(valueChanged(int)),this,SLOT(updateQueueSize(int))) ;
QObject::connect(ui._minPrioritized_SB,SIGNAL(valueChanged(int)),this,SLOT(updateMinPrioritized(int))) ;
QObject::connect(ui._defaultStrategy_CB,SIGNAL(activated(int)),this,SLOT(updateDefaultStrategy(int))) ;
QObject::connect(ui._diskSpaceLimit_SB,SIGNAL(valueChanged(int)),this,SLOT(updateDiskSizeLimit(int))) ;
QObject::connect(ui._max_tr_up_per_sec_SB, SIGNAL( valueChanged( int ) ), this, SLOT( updateMaxTRUpRate(int) ) );
@ -80,15 +78,7 @@ void TransferPage::updateDiskSizeLimit(int s)
rsFiles->setFreeDiskSpaceLimit(s) ;
}
void TransferPage::updateMinPrioritized(int s)
{
rsFiles->setMinPrioritizedTransfers(s) ;
}
void TransferPage::updateQueueSize(int s)
{
if(ui._minPrioritized_SB->value() > s)
{
ui._minPrioritized_SB->setValue(s) ;
}
rsFiles->setQueueSize(s) ;
}

1
retroshare-gui/src/gui/settings/TransferPage.h
View File

@ -46,7 +46,6 @@ class TransferPage: public ConfigPage
public slots:
void updateQueueSize(int) ;
void updateMinPrioritized(int) ;
void updateDefaultStrategy(int) ;
void updateDiskSizeLimit(int) ;
void updateMaxTRUpRate(int);

36
retroshare-gui/src/gui/settings/TransferPage.ui
View File

@ -6,8 +6,8 @@
<rect>
<x>0</x>
<y>0</y>
<width>604</width>
<height>340</height>
<width>700</width>
<height>356</height>
</rect>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
@ -28,13 +28,6 @@
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="label_4">
<property name="text">
<string>Slots reserved for non-cache transfers:</string>
</property>
</widget>
</item>
<item>
<widget class="QLabel" name="label_3">
<property name="text">
@ -76,19 +69,6 @@
</property>
</widget>
</item>
<item>
<widget class="QSpinBox" name="_minPrioritized_SB">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;You can use this to force RetroShare to download your files rather &lt;br/&gt;than cache files for as many slots as requested. Setting that number &lt;br/&gt;to be equal to the queue size above will always prioritize your files&lt;br/&gt;over cache. &lt;br/&gt;&lt;br/&gt;It is however recommended to leave at least a few slots for cache files. For now, cache files are only used to transfer friend file lists.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="minimum">
<number>1</number>
</property>
<property name="value">
<number>3</number>
</property>
</widget>
</item>
<item>
<widget class="QComboBox" name="_defaultStrategy_CB">
<property name="enabled">
@ -168,12 +148,12 @@
<string>&lt;!DOCTYPE HTML PUBLIC &quot;-//W3C//DTD HTML 4.0//EN&quot; &quot;http://www.w3.org/TR/REC-html40/strict.dtd&quot;&gt;
&lt;html&gt;&lt;head&gt;&lt;meta name=&quot;qrichtext&quot; content=&quot;1&quot; /&gt;&lt;style type=&quot;text/css&quot;&gt;
p, li { white-space: pre-wrap; }
&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Ubuntu'; font-size:11pt; font-weight:400; font-style:normal;&quot;&gt;
&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt; font-weight:600;&quot;&gt;RetroShare&lt;/span&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt;&quot;&gt; is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.&lt;/span&gt;&lt;/p&gt;
&lt;p style=&quot;-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-family:'Sans'; font-size:8pt;&quot;&gt;&lt;br /&gt;&lt;/p&gt;
&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-family:'Sans'; font-size:8pt;&quot;&gt;You can separately setup share flags for each shared directory in the shared files dialog to be:&lt;/span&gt;&lt;/p&gt;
&lt;ul style=&quot;margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;&quot;&gt;&lt;li style=&quot; font-family:'Sans'; font-size:8pt;&quot; style=&quot; margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Browsable by friends&lt;/span&gt;: files are seen by your friends.&lt;/li&gt;
&lt;li style=&quot; font-family:'Sans'; font-size:8pt;&quot; style=&quot; margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Anonymously shared&lt;/span&gt;: files are anonymously reachable through distant F2F tunnels.&lt;/li&gt;&lt;/ul&gt;&lt;/body&gt;&lt;/html&gt;</string>
&lt;/style&gt;&lt;/head&gt;&lt;body style=&quot; font-family:'Sans'; font-size:9pt; font-weight:400; font-style:normal;&quot;&gt;
&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-size:8pt; font-weight:600;&quot;&gt;RetroShare&lt;/span&gt;&lt;span style=&quot; font-size:8pt;&quot;&gt; is capable of transferring data and search requests between peers that are not necessarily friends. This traffic however only transits through a connected list of friends and is anonymous.&lt;/span&gt;&lt;/p&gt;
&lt;p style=&quot;-qt-paragraph-type:empty; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px; font-size:8pt;&quot;&gt;&lt;br /&gt;&lt;/p&gt;
&lt;p style=&quot; margin-top:0px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-size:8pt;&quot;&gt;You can separately setup share flags for each shared directory in the shared files dialog to be:&lt;/span&gt;&lt;/p&gt;
&lt;ul style=&quot;margin-top: 0px; margin-bottom: 0px; margin-left: 0px; margin-right: 0px; -qt-list-indent: 1;&quot;&gt;&lt;li style=&quot; font-size:8pt;&quot; style=&quot; margin-top:12px; margin-bottom:0px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Browsable by friends&lt;/span&gt;: files are seen by your friends.&lt;/li&gt;
&lt;li style=&quot; font-size:8pt;&quot; style=&quot; margin-top:0px; margin-bottom:12px; margin-left:0px; margin-right:0px; -qt-block-indent:0; text-indent:0px;&quot;&gt;&lt;span style=&quot; font-weight:600;&quot;&gt;Anonymously shared&lt;/span&gt;: files are anonymously reachable through distant F2F tunnels.&lt;/li&gt;&lt;/ul&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
</widget>
</item>

2
retroshare-gui/src/main.cpp
View File

@ -382,7 +382,7 @@ feenableexcept(FE_INVALID | FE_DIVBYZERO);
QObject::connect(notify,SIGNAL(deferredSignatureHandlingRequested()),notify,SLOT(handleSignatureEvent()),Qt::QueuedConnection) ;
QObject::connect(notify,SIGNAL(chatLobbyTimeShift(int)),notify,SLOT(handleChatLobbyTimeShift(int)),Qt::QueuedConnection) ;
QObject::connect(notify,SIGNAL(diskFull(int,int)) ,w ,SLOT(displayDiskSpaceWarning(int,int))) ;
QObject::connect(notify,SIGNAL(filesPostModChanged(bool)) ,w ,SLOT(postModDirectories(bool) )) ;
QObject::connect(notify,SIGNAL(filesPostModChanged(bool)) ,w ,SLOT(postModDirectories(bool)) ,Qt::QueuedConnection ) ;
QObject::connect(notify,SIGNAL(transfersChanged()) ,w->transfersDialog ,SLOT(insertTransfers() )) ;
QObject::connect(notify,SIGNAL(publicChatChanged(int)) ,w->friendsDialog ,SLOT(publicChatChanged(int) ));
QObject::connect(notify,SIGNAL(neighboursChanged()) ,w->friendsDialog->networkDialog ,SLOT(securedUpdateDisplay())) ;

2
retroshare-nogui/src/introserver.cc
View File

@ -290,7 +290,7 @@ int RsIntroServer::checkForNewCerts()
mCertCheckTime = now;
struct stat64 buf;
while(dirIt.readdir())
for(;dirIt.isValid();dirIt.next())
{
/* check entry type */
std::string fname;

6
tests/unittests/unittests.pro
View File

@ -353,9 +353,9 @@ SOURCES += libretroshare/gxs/data_service/rsdataservice_test.cc \
################################ dbase #####################################
SOURCES += libretroshare/dbase/fisavetest.cc \
libretroshare/dbase/fitest2.cc \
libretroshare/dbase/searchtest.cc \
#SOURCES += libretroshare/dbase/fisavetest.cc \
# libretroshare/dbase/fitest2.cc \
# libretroshare/dbase/searchtest.cc \
# libretroshare/dbase/ficachetest.cc \
# libretroshare/dbase/fimontest.cc \