mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-04-11 18:29:16 -04:00
added interface for p3FileLists
This commit is contained in:
mr-alice
parent
8e2ff56f9a
commit
a6d467d138
libretroshare/src/file_sharing
@ -40,9 +40,35 @@ Big picture
|
||||
- the same file should be able to be held by two different directories. Hash search will return a single hit.
|
||||
- the previously existing PersonEntry had no field and was overloading DirEntry, with overwritten file names, hashes etc. Super bad!
|
||||
|
||||
Directory storage file format
|
||||
-----------------------------
|
||||
* should be extensible (xml or binary format? Binary, because it's going to be encrypted anyway)
|
||||
=> works with binary fields
|
||||
=>
|
||||
|
||||
[= version =]
|
||||
[= peer id =]
|
||||
[= num entries =]
|
||||
[= some information =]
|
||||
|
||||
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
|
||||
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
|
||||
[entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01] [Field ID 02] [field size v 02] [Field data 02] ...
|
||||
...
|
||||
2 1-5 v 2 1-5 v
|
||||
* entry content
|
||||
Tag | Content | Size
|
||||
----------------+--------------------------------------+------
|
||||
01 | sha1 hash | 20
|
||||
01 | sha1^2 hash | 20
|
||||
02 | file name | < 512
|
||||
03 | file size | 8
|
||||
04 | dir name | < 512
|
||||
05 | last modif time local | 4
|
||||
06 | last modif time including sub-dirs | 4
|
||||
|
||||
Classes
|
||||
-------
|
||||
Rs
|
||||
|
||||
p3ShareManager
|
||||
- tick()
|
||||
@ -75,6 +101,29 @@ Classes
|
||||
- parent groups
|
||||
- group flags
|
||||
|
||||
Best data structure for file index
|
||||
----------------------------------
|
||||
|
||||
| Hash map map list
|
||||
----------------+-----------------+------------+--------------
|
||||
Adding | Constant | log(n) | O(n)
|
||||
Hash search | Constant | log(n) | O(n)
|
||||
Name/exp search | O(n) | O(n) | O(n)
|
||||
Recursive browse| Constant | log(n) | O(n)
|
||||
|
||||
Should we use the same struct for files and directories?
|
||||
|
||||
Sol 1:
|
||||
DirClass + PersonClass + FileEntry class
|
||||
- each has pointers to elements list of the same type
|
||||
- lists are handled for Files (all file entries),
|
||||
|
||||
Directories are represented by the hash of the full path
|
||||
|
||||
Sol 2:
|
||||
Same class for all elements, in a single hash map. Each element is
|
||||
defined by its type (Dir, Person, File) which all have a hash.
|
||||
|
||||
Syncing between peers
|
||||
---------------------
|
||||
|
||||
@ -83,10 +132,18 @@ Generating sync events
|
||||
- for each directory, in breadth first order
|
||||
- if directory has changed, or last update is old
|
||||
=> push a sync request
|
||||
- store the peer's last up time. Compare with peer uptimes recursively.
|
||||
|
||||
* Server side
|
||||
- after a change, broadcast a "directory changed" packet to all connected friends
|
||||
|
||||
|
||||
* directoy updater
|
||||
- crawl through directories
|
||||
- compare TS of files, missing files, new files
|
||||
- feed a queue of files to hash
|
||||
- directory whatcher gets notified when files are hashed
|
||||
|
||||
- a separate component hashes files (FileHashingProcess)
|
||||
|
||||
DirectoryWatcher (watches a hierarchy) File List (stores a directory hierarchy)
|
||||
| |
|
||||
@ -100,7 +157,6 @@ Generating sync events
|
||||
+----------- own file list -------+---------- Encrypted/compressed save to disk
|
||||
| | |
|
||||
+----------- friend file lists ---+
|
||||
|
||||
Roadmap
|
||||
-------
|
||||
|
||||
@ -115,12 +171,29 @@ Roadmap
|
||||
|
||||
TODO
|
||||
====
|
||||
[ ] implement directory updater
|
||||
[ ] local directory updater
|
||||
[ ] remote directory updater
|
||||
|
||||
[ ] implement directory handler
|
||||
[ ] implement p3FileLists with minimal functonality: no exchange. Only storage of own
|
||||
[ ] directory handler
|
||||
[ ] abstract functions to keep a directory and get updates to it.
|
||||
[ ] hierarchical storage representation.
|
||||
[ ] allow add/delete entries
|
||||
[ ] auto-cleanup
|
||||
|
||||
[ ] directory updater
|
||||
[ ] abstract layer
|
||||
[ ] crawls the directory and ask updates
|
||||
|
||||
[ ] derive local directory updater
|
||||
[ ] crawl local files, and asks updates to storage class
|
||||
[ ] derive remote directory updater
|
||||
[ ] crawl stored files, and request updates to storage class
|
||||
|
||||
[ ] load/save of directory content. Should be extensible
|
||||
[ ] p3FileLists with minimal functonality: no exchange. Only storage of own file lists
|
||||
[ ] service (items) for p3FileLists
|
||||
[ ] connect RemoteDirModel to new system
|
||||
[ ] test GUI functions
|
||||
[ ] test update between peers
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@ -19,11 +19,68 @@
|
||||
// The file lists are not directry updated. A FileListWatcher class is responsible for this
|
||||
// in every case.
|
||||
//
|
||||
class p3FileLists: public p3Service
|
||||
#include <services/p3service.h>
|
||||
#include <pqi/p3cfgmgr.h>
|
||||
|
||||
class p3FileLists: public p3Service, public p3Config, public RsSharedFileService
|
||||
{
|
||||
public:
|
||||
p3FileLists() ;
|
||||
typedef uint64_t EntryIndex ; // this should probably be defined elsewhere
|
||||
|
||||
p3FileLists(mPeerMgr *mpeers) ;
|
||||
|
||||
/*
|
||||
*/
|
||||
|
||||
// derived from p3Service
|
||||
//
|
||||
virtual int tick() ;
|
||||
|
||||
// access to own/remote shared files
|
||||
//
|
||||
virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list<std::string>& parent_groups) const;
|
||||
|
||||
virtual int SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) ;
|
||||
virtual int SearchBoolExp(Expression *exp, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
|
||||
|
||||
// Interface for browsing dir hierarchy
|
||||
//
|
||||
int RequestDirDetails(EntryIndex, DirDetails&, FileSearchFlags) const ;
|
||||
uint32_t getType(const EntryIndex&) const ;
|
||||
int RequestDirDetails(const std::string& path, DirDetails &details) const ;
|
||||
|
||||
// set/update shared directories
|
||||
void setSharedDirectories(const std::list<SharedDirInfo>& dirs);
|
||||
void getSharedDirectories(std::list<SharedDirInfo>& dirs);
|
||||
void updateShareFlags(const SharedDirInfo& info) ;
|
||||
|
||||
void forceDirectoryCheck(); // Force re-sweep the directories and see what's changed
|
||||
bool inDirectoryCheck();
|
||||
|
||||
// Derived from p3Config
|
||||
//
|
||||
private:
|
||||
p3PeerMgr *mPeers ;
|
||||
|
||||
// File sync request queues. The fast one is used for online browsing when friends are connected.
|
||||
// The slow one is used for background update of file lists.
|
||||
//
|
||||
std::list<RsFileListSyncRequest> mFastRequestQueue ;
|
||||
std::list<RsFileListSyncRequest> mSlowRequestQueue ;
|
||||
|
||||
// Directory storage hierarchies
|
||||
//
|
||||
std::map<RsPeerId,RemoteDirectoryStorage *> mRemoteDirectories ;
|
||||
|
||||
LocalSharedDirectoryMap *mLocalSharedDirs ;
|
||||
|
||||
RemoteSharedDirectoryWatcher *mRemoteDirWatcher ;
|
||||
LocalSharedDirectoryWatcher *mLocalDirWatcher ;
|
||||
|
||||
// We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name.
|
||||
//
|
||||
HashCache *mHashCache ;
|
||||
|
||||
RsMutex mFLSMtx ;
|
||||
};
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user