added interface for p3FileLists

libretroshare/src/file_sharing/README.txt

@@ -40,9 +40,35 @@ Big picture
          - the same file should be able to be held by two different directories. Hash search will return a single hit.
          - the previously existing PersonEntry had no field and was overloading DirEntry, with overwritten file names, hashes etc. Super bad!
 
+Directory storage file format
+-----------------------------
+   * should be extensible (xml or binary format? Binary, because it's going to be encrypted anyway)
+      => works with binary fields
+      => 
+
+      [= version =]
+      [= peer id =]
+      [= num entries =]
+      [= some information =]
+
+      [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01]  [Field ID 02] [field size v 02] [Field data 02] ...
+      [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01]  [Field ID 02] [field size v 02] [Field data 02] ...
+      [entry tag] [entry size] [Field ID 01] [field size v 01] [Field data 01]  [Field ID 02] [field size v 02] [Field data 02] ...
+         ...
+                                     2             1-5                v               2             1-5               v
+   * entry content
+         Tag           | Content                              |  Size
+       ----------------+--------------------------------------+------
+         01            | sha1 hash                            |  20
+         01            | sha1^2 hash                          |  20
+         02            | file name                            |  < 512
+         03            | file size                            |  8
+         04            | dir name                             |  < 512
+         05            | last modif time local                |  4
+         06            | last modif time including sub-dirs   |  4
+
 Classes
 -------
-   Rs
 
    p3ShareManager
       - tick()
@@ -75,6 +101,29 @@ Classes
       - parent groups
       - group flags
 
+Best data structure for file index
+----------------------------------
+
+                      | Hash map          map          list
+      ----------------+-----------------+------------+--------------
+      Adding          | Constant        | log(n)     | O(n)             
+      Hash search     | Constant        | log(n)     | O(n)             
+      Name/exp search | O(n)            | O(n)       | O(n)
+      Recursive browse| Constant        | log(n)     | O(n)
+
+      Should we use the same struct for files and directories?
+
+      Sol 1:
+         DirClass + PersonClass + FileEntry class
+            - each has pointers to elements list of the same type
+            - lists are handled for Files (all file entries), 
+
+            Directories are represented by the hash of the full path 
+
+      Sol 2:
+         Same class for all elements, in a single hash map. Each element is
+         defined by its type (Dir, Person, File) which all have a hash.
+
 Syncing between peers
 ---------------------
 
@@ -83,10 +132,18 @@ Generating sync events
       - for each directory, in breadth first order
          - if directory has changed, or last update is old
             => push a sync request
+         - store the peer's last up time. Compare with peer uptimes recursively.
 
    * Server side
       - after a change, broadcast a "directory changed" packet to all connected friends
-   
+
+   * directoy updater
+      - crawl through directories
+         - compare TS of files, missing files, new files
+         - feed a queue of files to hash
+      - directory whatcher gets notified when files are hashed
+
+      - a separate component hashes files (FileHashingProcess)
 
    DirectoryWatcher (watches a hierarchy)        File List (stores a directory hierarchy)
          |                                          |
@@ -100,7 +157,6 @@ Generating sync events
                                  +----------- own file list -------+---------- Encrypted/compressed save to disk
                                  |                  |              |
                                  +----------- friend file lists ---+
-
 Roadmap
 -------
 
@@ -115,12 +171,29 @@ Roadmap
 
 TODO
 ====
-   [ ] implement directory updater
-         [ ] local directory updater
-         [ ] remote directory updater
 
-   [ ] implement directory handler
+   [ ] directory handler
-   [ ] implement p3FileLists with minimal functonality: no exchange. Only storage of own
+         [ ] abstract functions to keep a directory and get updates to it.
+         [ ] hierarchical storage representation.
+               [ ] allow add/delete entries
+               [ ] auto-cleanup
+
+   [ ] directory updater
+         [ ] abstract layer
+               [ ] crawls the directory and ask updates
+
+         [ ] derive local directory updater
+               [ ] crawl local files, and asks updates to storage class
+         [ ] derive remote directory updater
+               [ ] crawl stored files, and request updates to storage class
+
+   [ ] load/save of directory content. Should be extensible
+   [ ] p3FileLists with minimal functonality: no exchange. Only storage of own file lists
+   [ ] service (items) for p3FileLists
+   [ ] connect RemoteDirModel to new system
+   [ ] test GUI functions
+   [ ] test update between peers
+
 
 
 

libretroshare/src/file_sharing/p3filelists.h

@@ -19,11 +19,68 @@
 // The file lists are not directry updated. A FileListWatcher class is responsible for this
 // in every case. 
 //
-class p3FileLists: public p3Service
+#include <services/p3service.h>
+#include <pqi/p3cfgmgr.h>
+
+class p3FileLists: public p3Service, public p3Config, public RsSharedFileService 
 {
 	public:
-		p3FileLists() ;
+		typedef uint64_t EntryIndex ;	// this should probably be defined elsewhere
 
+		p3FileLists(mPeerMgr *mpeers) ;
 
+		/*   
+		 */
+		 
+		// derived from p3Service
+		//
+		virtual int tick() ;
+
+		// access to own/remote shared files
+		//
+		virtual bool findLocalFile(const RsFileHash& hash,FileSearchFlags flags,const RsPeerId& peer_id, std::string &fullpath, uint64_t &size,FileStorageFlags& storage_flags,std::list<std::string>& parent_groups) const;
+
+		virtual int SearchKeywords(const std::list<std::string>& keywords, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) ;
+		virtual int SearchBoolExp(Expression *exp, std::list<DirDetails>& results,FileSearchFlags flags,const RsPeerId& peer_id) const ;
+
+		// Interface for browsing dir hierarchy
+		//
+		int RequestDirDetails(EntryIndex, DirDetails&, FileSearchFlags) const ;
+		uint32_t getType(const EntryIndex&) const ;
+		int RequestDirDetails(const std::string& path, DirDetails &details) const ;
+
+      // set/update shared directories
+		void setSharedDirectories(const std::list<SharedDirInfo>& dirs);
+		void getSharedDirectories(std::list<SharedDirInfo>& dirs);
+		void updateShareFlags(const SharedDirInfo& info) ;
+
+		void forceDirectoryCheck();              // Force re-sweep the directories and see what's changed
+		bool inDirectoryCheck();
+
+		// Derived from p3Config
+		//
+	private:
+		p3PeerMgr *mPeers ;
+
+		// File sync request queues. The fast one is used for online browsing when friends are connected.
+		// The slow one is used for background update of file lists.
+		//
+		std::list<RsFileListSyncRequest> mFastRequestQueue ;
+		std::list<RsFileListSyncRequest> mSlowRequestQueue ;
+
+		// Directory storage hierarchies
+		//
+		std::map<RsPeerId,RemoteDirectoryStorage *> mRemoteDirectories ;
+		
+		LocalSharedDirectoryMap *mLocalSharedDirs ;
+
+		RemoteSharedDirectoryWatcher *mRemoteDirWatcher ;
+		LocalSharedDirectoryWatcher *mLocalDirWatcher ;
+
+		// We use a shared file cache as well, to avoid re-hashing files with known modification TS and equal name.
+		//
+		HashCache *mHashCache ;
+
+		RsMutex mFLSMtx ;
 };