Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-24 23:19:29 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

- Removed old file indexing code.

Browse Source 
- Switched file sizes from 32bits -> 64 bits so rs can handle large files.
- combined Cache/Service Ids are now located in serialiser/rsserviceids.h



git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@273 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
drbob 2007-12-12 01:08:31 +00:00
parent 470ceaa73b
commit 07d33009b9
15 changed files with 24 additions and 1355 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
libretroshare/src/dbase/Makefile
View File

@ -2,11 +2,10 @@
RS_TOP_DIR = .. RS_TOP_DIR = ..
include ../make.opt include ../make.opt
OBJ = filedex.o filelook.o \ OBJ = findex.o fimonitor.o cachestrapper.o fistore.o \
findex.o fimonitor.o cachestrapper.o fistore.o \
rsexpr.o rsexpr.o
EXEC = ftest looktest fitest2 fisavetest ficachetest searchtest EXEC = fitest2 fisavetest ficachetest searchtest
ifeq ($(OS),Linux) ifeq ($(OS),Linux)
EXEC += fimontest EXEC += fimontest

4
libretroshare/src/dbase/cachestrapper.cc
View File

@ -639,7 +639,7 @@ bool CacheTransfer::RequestCache(CacheData &data, CacheStore *cbStore)
/* to be overloaded */ /* to be overloaded */
bool CacheTransfer::RequestCacheFile(RsPeerId id, std::string path, std::string hash, uint32_t size) bool CacheTransfer::RequestCacheFile(RsPeerId id, std::string path, std::string hash, uint64_t size)
{ {
std::cerr << "CacheTransfer::RequestCacheFile() : from:" << id << " #"; std::cerr << "CacheTransfer::RequestCacheFile() : from:" << id << " #";
std::cerr << hash << " size: " << size; std::cerr << hash << " size: " << size;
@ -704,7 +704,7 @@ bool CacheTransfer::FailedCache(std::string hash)
} }
bool CacheTransfer::FindCacheFile(std::string hash, std::string &path, uint32_t &size) bool CacheTransfer::FindCacheFile(std::string hash, std::string &path, uint64_t &size)
{ {
CacheData data; CacheData data;
if (strapper->findCache(hash, data)) if (strapper->findCache(hash, data))

11
libretroshare/src/dbase/cachestrapper.h
View File

@ -74,11 +74,6 @@ class CacheId
bool operator<(const CacheId &a, const CacheId &b); bool operator<(const CacheId &a, const CacheId &b);
#define CACHE_TYPE_FILE_INDEX 1
#define CACHE_TYPE_WEB 2
#define CACHE_TYPE_OTHER 4
/* think of any others? */
class CacheData class CacheData
{ {
@ -90,7 +85,7 @@ class CacheData
std::string path; std::string path;
std::string name; std::string name;
std::string hash; std::string hash;
uint32_t size; uint64_t size;
time_t recvd; time_t recvd;
}; };
@ -106,7 +101,7 @@ class CacheTransfer
virtual ~CacheTransfer() {} virtual ~CacheTransfer() {}
/* upload side of things .... searches through CacheStrapper. */ /* upload side of things .... searches through CacheStrapper. */
bool FindCacheFile(std::string hash, std::string &path, uint32_t &size); bool FindCacheFile(std::string hash, std::string &path, uint64_t &size);
/* At the download side RequestCache() => overloaded RequestCacheFile() /* At the download side RequestCache() => overloaded RequestCacheFile()
@ -117,7 +112,7 @@ bool RequestCache(CacheData &data, CacheStore *cbStore); /* request from CacheSt
protected: protected:
/* to be overloaded */ /* to be overloaded */
virtual bool RequestCacheFile(RsPeerId id, std::string path, std::string hash, uint32_t size); virtual bool RequestCacheFile(RsPeerId id, std::string path, std::string hash, uint64_t size);
bool CompletedCache(std::string hash); /* internal completion -> does cb */ bool CompletedCache(std::string hash); /* internal completion -> does cb */
bool FailedCache(std::string hash); /* internal completion -> does cb */ bool FailedCache(std::string hash); /* internal completion -> does cb */

366
libretroshare/src/dbase/filedex.cc
View File

@ -1,366 +0,0 @@
/*
* "$Id: filedex.cc,v 1.8 2007-02-18 21:46:49 rmf24 Exp $"
*
* Other Bits for RetroShare.
*
* Copyright 2004-2006 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "filedex.h"
#include <iostream>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
fdex::fdex()
{
return;
}
fdex::fdex(const char *p, const char *d, const char *f, const char *e, int l, int v)
:path(p), subdir(d), file(f), ext(e), len(l), vis(v)
{
return;
}
int filedex::load(std::list<DirItem> dirs)
{
std::list<DirItem>::iterator it;
/* these must be done in the correct order to ensure
* that the visibility is correct
*/
for(int i = FD_VIS_EXACT_ONLY; i <= FD_VIS_LISTING; i++)
{
for(it = dirs.begin(); it != dirs.end(); it++)
{
if (i == it -> vis)
{
//std::cerr << "Adding Type(" << it -> vis << ") " << it -> basepath;
//std::cerr << std::endl;
dirtodo.push_back(*it);
}
}
}
return processdirs();
}
int filedex::clear()
{
std::list<fdex *>::iterator it;
dirtodo.clear();
dirdone.clear();
for(it = files.begin(); it != files.end(); it++)
{
delete(*it);
}
files.clear();
for(it = dirlist.begin(); it != dirlist.end(); it++)
{
delete(*it);
}
dirlist.clear();
return 1;
}
std::string strtolower2(std::string in)
{
std::string out = in;
for(int i = 0; i < (signed) out.length(); i++)
{
if (isupper(out[i]))
{
out[i] += 'a' - 'A';
}
}
return out;
}
std::string fileextension2(std::string in)
{
std::string out;
bool found = false;
for(int i = in.length() -1; (i >= 0) && (found == false); i--)
{
if (in[i] == '.')
{
found = true;
for(int j = i+1; j < (signed) in.length(); j++)
{
out += in[j];
}
}
}
return strtolower2(out);
}
std::list<fdex *> filedex::search(std::list<std::string> terms, int limit, bool local)
{
bool found;
std::list<fdex *> newlist;
int matches = 0;
std::list<fdex *>::iterator fit;
std::list<std::string>::iterator tit;
//std::cerr << "filedex::search() looking for" << std::endl;
//for(tit = terms.begin(); tit != terms.end(); tit++)
// std::cerr << "\t(" << *tit << ")" << std::endl;
//std::cerr << "Checking:" << std::endl;
for(fit = files.begin(); fit != files.end(); fit++)
{
// ignore named only files.
if ((!local) && ((*fit)->vis < FD_VIS_SEARCH))
continue;
found = true;
for(tit = terms.begin(); (tit != terms.end()) && (found); tit++)
{
std::string path = (*fit) -> subdir+"/"+(*fit)->file;
std::string lowerpath = strtolower2(path);
std::string lowerterm = strtolower2(*tit);
//std::cerr << "\tLooking for (" << lowerterm;
//std::cerr << ") in (" << lowerpath << ") ";
if (strstr(lowerpath.c_str(), lowerterm.c_str()) == NULL)
{
found = false;
//std::cerr << "\tFalse..." << std::endl;
}
else
{
//std::cerr << "\tTrue..." << std::endl;
}
}
if (found)
{
//std::cerr << "Found Matching!" << std::endl;
newlist.push_back(*fit);
if (++matches == limit)
{
//std::cerr << "Reached Limit(" << limit << ")";
//std::cerr << "Returning Results" << std::endl;
return newlist;
}
}
}
return newlist;
}
std::list<fdex *> filedex::dirlisting(std::string basedir)
{
std::list<fdex *> newlist;
std::list<fdex *>::iterator fit;
//std::cerr << "filedex::dirlisting() looking for subdir: " << basedir << std::endl;
//std::cerr << "Checking:" << std::endl;
for(fit = dirlist.begin(); fit != dirlist.end(); fit++)
{
//std::cerr << "DCHK(" << basedir << ") vs (" << (*fit) -> subdir << ")" << std::endl;
if (basedir == (*fit) -> subdir)
{
/* in the dir */
//std::cerr << "Found Matching SubDir:";
//std::cerr << (*fit) -> subdir << std::endl;
newlist.push_back(*fit);
}
}
for(fit = files.begin(); fit != files.end(); fit++)
{
//std::cerr << "FCHK(" << basedir << ") vs (" << (*fit) -> subdir << ")" << std::endl;
if (basedir == (*fit) -> subdir)
{
/* in the dir */
//std::cerr << "Found Matching File:";
//std::cerr << (*fit) -> subdir << std::endl;
newlist.push_back(*fit);
}
}
return newlist;
}
std::list<fdex *> filedex::findfilename(std::string name)
{
std::list<fdex *> newlist;
std::list<fdex *>::iterator fit;
std::cerr << "filedex::findfilename() looking for: " << name << std::endl;
std::cerr << "Checking:" << std::endl;
for(fit = files.begin(); fit != files.end(); fit++)
{
if (name == (*fit) -> file)
{
/* in the dir */
std::cerr << "Found Matching File!" << std::endl;
newlist.push_back(*fit);
}
}
return newlist;
}
int filedex::processdirs()
{
std::list<DirItem>::iterator it;
std::list<std::string>::iterator sit;
std::string dirname;
std::string subdir;
std::string fname;
std::string fullname;
struct dirent *ent;
struct stat buf;
bool found = false;
fdex *fx;
int count = 0;
int ficount = 0;
while(dirtodo.size() > 0)
{
count++;
it = dirtodo.begin();
DirItem currDir(*it);
dirname = currDir.getFullPath();
dirtodo.erase(it);
//std::cerr << "\tExamining: " << currDir.basepath;
//std::cerr << " -/- " << currDir.subdir << std::endl;
//std::cerr << "\t\t" << count << " Directories done, ";
//std::cerr << dirtodo.size() << " to go! " << std::endl;
// open directory.
DIR *dir = opendir(dirname.c_str());
if (dir != NULL) {
// read the directory.
while(NULL != (ent = readdir(dir)))
{
fname = ent -> d_name;
fullname = dirname + "/" + fname;
subdir = currDir.subdir;
// std::cerr << "Statting dirent: " << fullname.c_str() <<std::endl;
// std::cerr << "Statting dirent: " << fullname <<std::endl;
// stat each file.
if (-1 != stat(fullname.c_str(), &buf))
{
//std::cerr << "buf.st_mode: " << buf.st_mode <<std::endl;
if (S_ISDIR(buf.st_mode))
{
//std::cerr << "Is Directory: " << fullname << std::endl;
found = false;
// remove "." and ".." entries.
if ((fname == ".") || (fname == ".."))
{
found = true;
}
// check if in list of done already.
for(it = dirtodo.begin(); it != dirtodo.end(); it++)
{
if (it -> getFullPath() == fullname)
found = true;
}
for(sit = dirdone.begin(); sit != dirdone.end(); sit++)
{
if ((*sit) == fullname)
found = true;
}
if (found == false)
{
// add to list to read.
DirItem ndir(currDir.basepath, subdir+"/"+fname, currDir.vis);
// Push to the front of the list -> so processing done in order.
dirtodo.push_front(ndir);
// std::cerr << "\t Adding Directory: " << fullname;
// std::cerr << " to dirtodo" << std::endl;
if (currDir.vis == FD_VIS_LISTING)
{
// add to dirlist dbase.
fx = new fdex(fullname.c_str(), subdir.c_str(),
fname.c_str(), "DIR", 0, currDir.vis);
dirlist.push_back(fx);
// std::cerr << "\t Adding Directory: " << fullname;
// std::cerr << " to dirlist" << std::endl;
}
}
}
else if (S_ISREG(buf.st_mode))
{
// add to dbase.
fx = new fdex(fullname.c_str(), subdir.c_str(),
fname.c_str(), (fileextension2(fname)).c_str(),
buf.st_size, currDir.vis);
//fx -> path = fullname;
files.push_back(fx);
ficount++;
if (ficount % 100 == 0)
{
//std::cerr << "\tIndex(" << ficount << ") : ";
//std::cerr << "\tIndexing File: " << fname;
//std::cerr << std::endl;
}
}
else
{
// warning.
std::cerr << "\t Ignoring Unknown FileType";
std::cerr << std::endl;
}
}
else
{
}
}
// should check error message.
dirdone.push_back(dirname);
closedir(dir);
}
else
{
//std::cerr << "Cannot Open Directory:" << dirname << std::endl;
}
}
return 1;
}

111
libretroshare/src/dbase/filedex.h
View File

@ -1,111 +0,0 @@
/*
* "$Id: filedex.h,v 1.4 2007-02-18 21:46:49 rmf24 Exp $"
*
* Other Bits for RetroShare.
*
* Copyright 2004-2006 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_PQI_FILEINDEXER
#define MRK_PQI_FILEINDEXER
/* a file index.
*
* initiated with a list of directories.
*
*/
#include <list>
#include <string>
#define FD_VIS_EXACT_ONLY 0
#define FD_VIS_SEARCH 1
#define FD_VIS_LISTING 2
/* if vis = FD_VIS_EXACT_ONLY
* Then only an exact filename match works
* if vis = FD_VIS_SEARCH
* Then only a search will return it.
* if vis = FD_VIS_LISTING
* Then any method will find it.
*/
class fdex
{
public:
fdex();
fdex(const char *path, const char *srchpath,
const char *file, const char *ext, int len, int vis);
std::string path; // full path. (not searched)
std::string subdir; // searched if flag set.
std::string file;
std::string ext;
int len;
int vis;
};
class DirItem
{
public:
DirItem(std::string bp, std::string sd, int v)
: basepath(bp), subdir(sd), vis(v) {}
std::string getFullPath()
{
return basepath + subdir;
}
std::string basepath;
std::string subdir;
int vis;
};
class filedex
{
public:
int load(std::list<DirItem> dirs);
int clear();
// -1 for no limit.
// If remote ... then restrictions apply.
std::list<fdex *> search(std::list<std::string>, int limit, bool local);
std::list<fdex *> dirlisting(std::string);
std::list<fdex *> findfilename(std::string);
private:
int processdirs();
std::list<DirItem> dirtodo;
std::list<std::string> dirdone;
/* dbase */
std::list<fdex *> files;
std::list<fdex *> dirlist;
};
#endif

538
libretroshare/src/dbase/filelook.cc
View File

@ -1,538 +0,0 @@
/*
* "$Id: filelook.cc,v 1.3 2007-03-05 21:26:03 rmf24 Exp $"
*
* Other Bits for RetroShare.
*
* Copyright 2004-2006 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "dbase/filelook.h"
#include "util/rsdir.h"
#include <sstream>
#include <sys/types.h>
#include <sys/stat.h>
#include <unistd.h>
#include <dirent.h>
/*
* The Basic Low-Level Local File/Directory Interface.
*
* Requests are queued, and serviced (slowly)
*
*
* So this is a simple
* Local Lookup service,
* with a locked threaded interface.
*
* Input:
* "lookup directory".
*
* Output:
* "FileItems"
* "DirItems"
*
*
* a file index.
*
* initiated with a list of directories.
*
*/
fileLook::fileLook()
{
return;
}
fileLook::~fileLook()
{
return;
}
/* interface */
void fileLook::setSharedDirectories(std::list<std::string> dirs)
{
int i;
lockDirs(); { /* LOCKED DIRS */
/* clear old directories */
sharedDirs.clear();
/* iterate through the directories */
std::list<std::string>::iterator it;
for(it = dirs.begin(); it != dirs.end(); it++)
{
/* get the head directory */
std::string root_dir = *it;
std::string top_dir = RsDirUtil::getTopDir(root_dir);
/* if unique -> add, else add modifier */
bool unique = false;
for(i = 0; !unique; i++)
{
std::string tst_dir = top_dir;
if (i > 0)
{
std::ostringstream out;
out << "-" << i;
tst_dir += out.str();
}
std::map<std::string, std::string>::const_iterator cit;
if (sharedDirs.end()== (cit=sharedDirs.find(tst_dir)))
{
unique = true;
/* add it! */
sharedDirs[tst_dir.c_str()] = root_dir;
std::cerr << "Added [" << tst_dir << "] => " << root_dir << std::endl;
}
}
}
} unlockDirs(); /* UNLOCKED DIRS */
}
int fileLook::lookUpDirectory(PQItem *i)
{
std::cerr << "lookUpDirectory() About to Lock" << std::endl;
lockQueues();
std::cerr << "lookUpDirectory():" << std::endl;
i -> print(std::cerr);
mIncoming.push_back(i);
unlockQueues();
std::cerr << "lookUpDirectory() Unlocked" << std::endl;
return 1;
}
PQItem *fileLook::getResult()
{
PQItem *i = NULL;
lockQueues();
if (mOutgoing.size() > 0)
{
i = mOutgoing.front();
mOutgoing.pop_front();
std::cerr << "getResult()" << std::endl;
//i -> print(std::cerr);
}
unlockQueues();
return i;
}
/* locking
void fileLook::lockQueues()
{}
void fileLook::unlockQueues()
{}
void fileLook::lockDirs()
{}
void fileLook::unlockDirs()
{}
************/
void fileLook::run()
{
std::cerr << "fileLook::run() started." << std::endl;
processQueue();
}
void fileLook::processQueue()
{
while(1)
{
PQItem *i = NULL;
//std::cerr << "fileLook::run() -> lockQueues()" << std::endl;
lockQueues();
if (mIncoming.size() > 0)
{
i = mIncoming.front();
mIncoming.pop_front();
}
unlockQueues();
//std::cerr << "fileLook::run() -> unlockQueues()" << std::endl;
if (i)
{
/* process item */
/* look up the directory */
std::cerr << "fileLook::run() -> loadDirectory()" << std::endl;
loadDirectory(i);
}
else
{
/* sleep */
//std::cerr << "fileLook::sleep() ;)" << std::endl;
#ifndef WINDOWS_SYS /* UNIX */
usleep(50000); /* 50 milli secs (1/20)th of sec */
#else
Sleep(50); /* 50 milli secs (1/20)th of sec */
#endif
}
}
}
/* THIS IS THE ONLY BIT WHICH DEPENDS ON PQITEM STRUCTURE */
void fileLook::loadDirectory(PQItem *dir)
{
/* extract the path */
std::cerr << "loadDirectory()" << std::endl;
/* check its a PQFileItem */
PQFileItem *fi = dynamic_cast<PQFileItem *>(dir);
if (!fi)
{
return;
}
/* we need the root directory, and the rest */
std::string req_dir = fi -> path;
std::string root_dir = RsDirUtil::getRootDir(req_dir);
std::string rest_path = RsDirUtil::removeRootDirs(req_dir, root_dir);
/* get directory listing */
std::string full_root = "/dev/null";
if (root_dir == "")
{
std::cerr << "loadDirectory() Root Request" << std::endl;
loadRootDirs(fi);
}
int err = 0;
lockDirs();
{
/* check that dir is one of the shared */
std::map<std::string, std::string>::iterator it;
it = sharedDirs.find(root_dir.c_str());
if (it == sharedDirs.end())
{
err = 1;
}
else
{
full_root = it -> second;
}
}
unlockDirs();
std::cerr << "root_dir = [" << root_dir << "]" << std::endl;
std::cerr << "full_root = " << full_root << std::endl;
if (err)
{
std::cerr << "Rejected: root_dir = " << root_dir << std::endl;
for(unsigned int i = 0; i < root_dir.length(); i++)
{
std::cerr << "[" << (int) root_dir[i] << "]:[" << (int) root_dir[i] << "]" << std::endl;
}
std::map<std::string, std::string>::iterator it;
for(it = sharedDirs.begin(); it != sharedDirs.end(); it++)
{
std::cerr << "Possible Root: " << it -> first << std::endl;
}
return;
}
/* otherwise load directory */
std::string full_path = full_root;
if (rest_path.length() > 0)
{
full_path += "/" + rest_path;
}
std::cerr << "Looking up path:" << full_path << std::endl;
/* lookup base dir */
DIR *rootdir = opendir(full_path.c_str());
struct dirent *rootent, *subent;
struct stat buf;
if (rootdir == NULL) {
/* error */
std::cerr << "Error in Path" << std::endl;
return;
}
std::list<PQItem *> subitems;
int rootcount = 0;
while(NULL != (rootent = readdir(rootdir)))
{
/* iterate through the entries */
bool issubdir = false;
/* check entry type */
std::string fname = rootent -> d_name;
std::string fullname = full_path + "/" + fname;
std::cerr << "Statting dirent: " << fullname <<std::endl;
if (-1 != stat(fullname.c_str(), &buf))
{
std::cerr << "buf.st_mode: " << buf.st_mode <<std::endl;
if (S_ISDIR(buf.st_mode))
{
std::cerr << "Is Directory: " << fullname << std::endl;
if ((fname == ".") || (fname == ".."))
{
std::cerr << "Skipping:" << fname << std::endl;
continue; /* skipping links */
}
issubdir = true;
}
else if (S_ISREG(buf.st_mode))
{
/* is file */
std::cerr << "Is File: " << fullname << std::endl;
issubdir = false;
}
else
{
/* unknown , ignore */
continue;
}
}
/* If subdir - Count the subentries */
DIR *subdir = NULL;
if (issubdir)
{
subdir = opendir(fullname.c_str());
}
int subdirno = 0;
if (subdir)
{
while(NULL != (subent = readdir(subdir)))
{
std::string fname = subent -> d_name;
if (
#ifndef WINDOWS_SYS /* UNIX */
(DT_DIR == subent -> d_type) && /* Only works under Unix */
#else /* WIndows */
#endif
((fname == ".") || (fname == "..")))
{
/* do not add! */
std::cerr << "Removed entry:" << fname << " from subdir count" << std::endl;
}
else
{
subdirno++;
}
}
/* clean up dir */
closedir(subdir);
}
rootcount++;
std::string subname = rootent -> d_name;
/* create an item */
PQFileItem *finew = fi -> clone();
// path should be copied auto.
finew -> name = subname;
if (issubdir)
{
finew -> reqtype = PQIFILE_DIRLIST_DIR;
finew -> size = subdirno; /* number of entries in dir */
}
else
{
finew -> reqtype = PQIFILE_DIRLIST_FILE;
finew -> size = buf.st_size;
}
subitems.push_back(finew);
}
/* clean up dir */
closedir(rootdir);
/* if necessary create Root Entry */
/* if constructing Tree - do it here */
/* put on queue */
lockQueues();
std::list<PQItem *>::iterator it;
for(it = subitems.begin(); it != subitems.end(); it++)
{
/* push onto outgoing queue */
mOutgoing.push_back(*it);
}
subitems.clear();
unlockQueues();
};
void fileLook::loadRootDirs(PQFileItem *dir)
{
/* extract the path */
std::cerr << "loadRootDirs()" << std::endl;
dir -> path = "";
int rootcount = 0;
lockDirs();
std::list<PQItem *> subitems;
std::map<std::string, std::string>::iterator it;
for(it = sharedDirs.begin(); it != sharedDirs.end(); it++)
{
/* lookup base dir */
DIR *entdir = opendir(it -> second.c_str());
struct dirent *subent;
if (!entdir)
{
std::cerr << "Bad Shared Dir: " << it->second << std::endl;
continue;
}
int subdirno = 0;
while(NULL != (subent = readdir(entdir)))
{
std::string fname = subent -> d_name;
if (
#ifndef WINDOWS_SYS /* UNIX */
(DT_DIR == subent -> d_type) && /* Only works under Unix */
#else /* WIndows */
#endif
((fname == ".") || (fname == "..")))
{
/* do not add! */
std::cerr << "Removed entry:" << fname << " from subdir count" << std::endl;
}
else
{
subdirno++;
}
}
closedir(entdir);
rootcount++;
/* create an item */
PQFileItem *finew = dir -> clone();
// path should be copied auto.
finew -> name = it -> first;
finew -> reqtype = PQIFILE_DIRLIST_DIR;
finew -> size = subdirno; /* number of entries in dir */
subitems.push_back(finew);
}
unlockDirs();
/* put on queue */
lockQueues();
std::list<PQItem *>::iterator it2;
for(it2 = subitems.begin(); it2 != subitems.end(); it2++)
{
/* push onto outgoing queue */
mOutgoing.push_back(*it2);
}
subitems.clear();
unlockQueues();
};
PQFileItem *fileLook::findFileEntry(PQFileItem *fi)
{
/* extract the path */
std::cerr << "findFileEntry()" << std::endl;
/* we need the root directory, and the rest */
std::string req_dir = fi -> path;
std::string root_dir = RsDirUtil::getRootDir(req_dir);
std::string rest_path = RsDirUtil::removeRootDirs(req_dir, root_dir);
/* get directory listing */
std::string full_root = "/dev/null";
int err = 0;
lockDirs();
{
/* check that dir is one of the shared */
std::map<std::string, std::string>::iterator it;
it = sharedDirs.find(root_dir.c_str());
if (it == sharedDirs.end())
{
err = 1;
}
else
{
full_root = it -> second;
}
}
unlockDirs();
PQFileItem *rtn = NULL;
if (err)
return rtn;
rtn = fi -> clone();
rtn -> path = full_root + "/" + rest_path;
return rtn;
}

113
libretroshare/src/dbase/filelook.h
View File

@ -1,113 +0,0 @@
/*
* "$Id: filelook.h,v 1.1 2007-02-19 20:08:30 rmf24 Exp $"
*
* Other Bits for RetroShare.
*
* Copyright 2004-2006 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#ifndef MRK_PQI_FILELOOK
#define MRK_PQI_FILELOOK
/*
* The Basic Low-Level Local File/Directory Interface.
*
* Requests are queued, and serviced (slowly)
*
*
* So this is a simple
* Local Lookup service,
* with a locked threaded interface.
*
* Input:
* "lookup directory".
*
* Output:
* "FileItems"
* "DirItems"
*
*
* a file index.
*
* initiated with a list of directories.
*
*/
#include <list>
#include <map>
#include "pqi/pqi.h"
#include "util/rsthreads.h"
class fileLook: public RsThread
{
public:
fileLook();
virtual ~fileLook();
/* threading */
virtual void run(); /* overloaded */
/* interface */
void setSharedDirectories(std::list<std::string> dirs);
/* I/O */
int lookUpDirectory(PQItem *i);
/* This one goes directly -> for speed, actually just a basepath lookup (no file check) */
PQFileItem *findFileEntry(PQFileItem *);
PQItem *getResult();
private:
/* locking */
void lockQueues() { qMtx.lock(); }
void unlockQueues() { qMtx.unlock(); }
void lockDirs() { dMtx.lock(); }
void unlockDirs() { dMtx.unlock(); }
RsMutex qMtx;
RsMutex dMtx;
/* threading queues */
std::list<PQItem *> mIncoming;
std::list<PQItem *> mOutgoing;
std::map<std::string, std::string> sharedDirs;
/* Processing Fns */
void processQueue();
/* THIS IS THE ONLY BIT WHICH DEPENDS ON PQITEM STRUCTURE */
void loadDirectory(PQItem *dir);
void loadRootDirs(PQFileItem *dir);
};
#endif

8
libretroshare/src/dbase/fimonitor.cc
View File

@ -23,6 +23,8 @@
#include "dbase/fimonitor.h" #include "dbase/fimonitor.h"
#include "util/rsdir.h" #include "util/rsdir.h"
#include "serialiser/rsserviceids.h"
#include <iostream> #include <iostream>
#include <sstream> #include <sstream>
#include <iomanip> #include <iomanip>
@ -37,7 +39,7 @@
#define FIM_DEBUG 1 #define FIM_DEBUG 1
FileIndexMonitor::FileIndexMonitor(std::string cachedir, std::string pid) FileIndexMonitor::FileIndexMonitor(std::string cachedir, std::string pid)
:CacheSource(CACHE_TYPE_FILE_INDEX, false, cachedir), fi(pid), :CacheSource(RS_SERVICE_TYPE_FILE_INDEX, false, cachedir), fi(pid),
pendingDirs(false), pendingForceCacheWrite(false) pendingDirs(false), pendingForceCacheWrite(false)
{ {
@ -52,7 +54,7 @@ FileIndexMonitor::~FileIndexMonitor()
} }
bool FileIndexMonitor::findLocalFile(std::string hash, bool FileIndexMonitor::findLocalFile(std::string hash,
std::string &fullpath, uint32_t &size) std::string &fullpath, uint64_t &size)
{ {
std::list<FileEntry *> results; std::list<FileEntry *> results;
bool ok = false; bool ok = false;
@ -494,7 +496,7 @@ void FileIndexMonitor::updateCycle()
#endif #endif
std::string calchash; std::string calchash;
uint32_t size; uint64_t size;
fi.saveIndex(fname, calchash, size); fi.saveIndex(fname, calchash, size);

2
libretroshare/src/dbase/fimonitor.h
View File

@ -72,7 +72,7 @@ class FileIndexMonitor: public CacheSource, public RsThread
virtual ~FileIndexMonitor(); virtual ~FileIndexMonitor();
/* external interface for filetransfer */ /* external interface for filetransfer */
bool findLocalFile(std::string hash, std::string &fullpath, uint32_t &size); bool findLocalFile(std::string hash, std::string &fullpath, uint64_t &size);
/* Interacting with CacheSource */ /* Interacting with CacheSource */

6
libretroshare/src/dbase/findex.cc
View File

@ -648,7 +648,7 @@ int FileIndex::printFileIndex(std::ostream &out)
} }
int FileIndex::loadIndex(std::string filename, std::string expectedHash, uint32_t size) int FileIndex::loadIndex(std::string filename, std::string expectedHash, uint64_t size)
{ {
std::ifstream file (filename.c_str(), std::ifstream::binary); std::ifstream file (filename.c_str(), std::ifstream::binary);
if (!file) if (!file)
@ -794,7 +794,7 @@ int FileIndex::loadIndex(std::string filename, std::string expectedHash, uint32_
nfile = new FileEntry(); nfile = new FileEntry();
nfile->name = tokens[0]; nfile->name = tokens[0];
nfile->hash = tokens[1]; nfile->hash = tokens[1];
nfile->size = atoi(tokens[2].c_str()); nfile->size = atoll(tokens[2].c_str());
nfile->modtime = atoi(tokens[3].c_str()); nfile->modtime = atoi(tokens[3].c_str());
nfile->pop = atoi(tokens[4].c_str()); nfile->pop = atoi(tokens[4].c_str());
nfile->updtime = atoi(tokens[5].c_str()); nfile->updtime = atoi(tokens[5].c_str());
@ -845,7 +845,7 @@ error:
} }
int FileIndex::saveIndex(std::string filename, std::string &fileHash, uint32_t &size) int FileIndex::saveIndex(std::string filename, std::string &fileHash, uint64_t &size)
{ {
unsigned char sha_buf[SHA_DIGEST_LENGTH]; unsigned char sha_buf[SHA_DIGEST_LENGTH];
std::ofstream file (filename.c_str(), std::ofstream::binary); std::ofstream file (filename.c_str(), std::ofstream::binary);

10
libretroshare/src/dbase/findex.h
View File

@ -84,11 +84,11 @@ virtual int print(std::ostream &out);
/* Data */ /* Data */
std::string name; std::string name;
std::string hash; std::string hash;
int size; /* file size */ uint64_t size; /* file size */
int modtime; /* modification time - most recent mod time for a sub entry for dirs */ time_t modtime; /* modification time - most recent mod time for a sub entry for dirs */
int pop; /* popularity rating */ int pop; /* popularity rating */
int updtime; /* last updated */ time_t updtime; /* last updated */
/* References for easy manipulation */ /* References for easy manipulation */
DirEntry *parent; DirEntry *parent;
@ -213,8 +213,8 @@ int cleanOldEntries(time_t old); /* removes entries older than old */
int printFileIndex(std::ostream &out); int printFileIndex(std::ostream &out);
/* load/save to file */ /* load/save to file */
int loadIndex(std::string filename, std::string expectedHash, uint32_t size); int loadIndex(std::string filename, std::string expectedHash, uint64_t size);
int saveIndex(std::string filename, std::string &fileHash, uint32_t &size); int saveIndex(std::string filename, std::string &fileHash, uint64_t &size);
/* search through this index */ /* search through this index */
int searchTerms(std::list<std::string> terms, std::list<FileEntry *> &results); int searchTerms(std::list<std::string> terms, std::list<FileEntry *> &results);

2
libretroshare/src/dbase/fisavetest.cc
View File

@ -33,7 +33,7 @@ int main()
fi1->printFileIndex(std::cout); fi1->printFileIndex(std::cout);
std::string fhash; std::string fhash;
uint32_t size; uint64_t size;
fi1->saveIndex("test.index", fhash, size); fi1->saveIndex("test.index", fhash, size);
std::cout << " Saved Index: Size: " << size << " Hash: " << fhash << std::endl; std::cout << " Saved Index: Size: " << size << " Hash: " << fhash << std::endl;

3
libretroshare/src/dbase/fistore.cc
View File

@ -23,10 +23,11 @@
#include "dbase/fistore.h" #include "dbase/fistore.h"
#include "rsiface/rsexpr.h" #include "rsiface/rsexpr.h"
#include "serialiser/rsserviceids.h"
FileIndexStore::FileIndexStore(CacheTransfer *cft, FileIndexStore::FileIndexStore(CacheTransfer *cft,
NotifyBase *cb_in, RsPeerId ownid, std::string cachedir) NotifyBase *cb_in, RsPeerId ownid, std::string cachedir)
:CacheStore(CACHE_TYPE_FILE_INDEX, false, cft, cachedir), :CacheStore(RS_SERVICE_TYPE_FILE_INDEX, false, cft, cachedir),
localId(ownid), localindex(NULL), cb(cb_in) localId(ownid), localindex(NULL), cb(cb_in)
{ {
return; return;

111
libretroshare/src/dbase/ftest.cc
View File

@ -1,111 +0,0 @@
/*
* "$Id: ftest.cc,v 1.5 2007-02-18 21:46:49 rmf24 Exp $"
*
* Other Bits for RetroShare.
*
* Copyright 2004-2006 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
/* Example Test of filedex index */
#include "filedex.h"
#include <iostream>
int main()
{
filedex fd;
std::string term;
std::list<DirItem> dirs;
std::list<std::string> terms;
std::list<fdex *> results;
std::list<fdex *>::iterator it;
dirs.push_back(DirItem("/usr/local", "", 1));
dirs.push_back(DirItem("/var/log", "", 2));
fd.load(dirs);
// now show the root directories...
results.clear();
results = fd.dirlisting("");
std::cerr << "Root Directory Results:" << std::endl;
for(it = results.begin(); it != results.end(); it++)
{
std::cerr << "Full Path: " << (*it) -> path << std::endl;
std::cerr << "\tFile:" << (*it) -> file << std::endl;
std::cerr << "\tExt:" << (*it) -> ext;
std::cerr << " Size: " << (*it) -> len << std::endl;
std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
}
while(1)
{
std::cerr << "Enter Search Term :";
std::cin >> term;
std::cerr << "Searching For:" << term << std::endl;
terms.clear();
terms.push_back(term);
results.clear();
results = fd.search(terms, 10, false);
std::cerr << "Search Results:" << std::endl;
for(it = results.begin(); it != results.end(); it++)
{
std::cerr << "Full Path: " << (*it) -> path << std::endl;
std::cerr << "\tFile:" << (*it) -> file << std::endl;
std::cerr << "\tExt:" << (*it) -> ext;
std::cerr << " Size: " << (*it) -> len << std::endl;
std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
}
results.clear();
results = fd.dirlisting(term);
std::cerr << "FileName Results:" << std::endl;
for(it = results.begin(); it != results.end(); it++)
{
std::cerr << "Full Path: " << (*it) -> path << std::endl;
std::cerr << "\tFile:" << (*it) -> file << std::endl;
std::cerr << "\tExt:" << (*it) -> ext;
std::cerr << " Size: " << (*it) -> len << std::endl;
std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
}
results.clear();
results = fd.findfilename(term);
std::cerr << "FileName Results:" << std::endl;
for(it = results.begin(); it != results.end(); it++)
{
std::cerr << "Full Path: " << (*it) -> path << std::endl;
std::cerr << "\tFile:" << (*it) -> file << std::endl;
std::cerr << "\tExt:" << (*it) -> ext;
std::cerr << " Size: " << (*it) -> len << std::endl;
std::cerr << "\tDir:" << (*it) -> subdir << std::endl;
}
}
//sleep(5);
return 1;
}

89
libretroshare/src/dbase/looktest.cc
View File

@ -1,89 +0,0 @@
/*
* "$Id: looktest.cc,v 1.1 2007-02-19 20:08:30 rmf24 Exp $"
*
* Other Bits for RetroShare.
*
* Copyright 2004-2006 by Robert Fernie.
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#include "filedex.h"
#include "dbase/filelook.h"
#include <iostream>
#include "pqi/pqi.h"
int main()
{
fileLook *fl = new fileLook();
std::string term;
std::list<std::string> dirs;
std::list<std::string> terms;
std::list<fdex *> results;
std::list<fdex *>::iterator it;
dirs.push_back("/mnt/disc2/extra/rmf24/mp3s/good");
dirs.push_back("/mnt/disc2/extra/rmf24/mp3s/marks");
dirs.push_back("/mnt/disc2/extra/rmf24/mp3s/incoming");
fl -> start(); /* background look thread */
std::cerr << "FileLook Thread started" << std::endl;
fl -> setSharedDirectories(dirs);
PQFileItem *i = new PQFileItem();
std::cerr << "test Lookup ..." << std::endl;
fl -> lookUpDirectory(i);
std::cerr << "Entering Main Loop" << std::endl;
while(1)
{
std::cerr << "Enter Search Term :";
std::cin >> term;
std::cerr << "Searching For:" << term << std::endl;
terms.clear();
terms.push_back(term);
PQFileItem *i = new PQFileItem();
i -> path = term;
std::cerr << "Root Lookup ..." << std::endl;
fl -> lookUpDirectory(i);
std::cerr << "LookUp done" << std::endl;
while(NULL != (i = (PQFileItem *) fl -> getResult()))
{
std::cerr << "Results:" << std::endl;
i -> print(std::cerr);
std::cerr << std::endl;
}
}
//sleep(5);
return 1;
}