mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-05-02 06:06:10 -04:00
Merge branch 'master' into jsonapi
This commit is contained in:
Gioacchino Mazzurco
commit
7787f9d62f
30 changed files with 782 additions and 68 deletions
|
|
@ -21,24 +21,25 @@
|
|||
******************************************************************************/
|
||||
#pragma once
|
||||
|
||||
static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes
|
||||
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes
|
||||
static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 sec. But we only update for real if something has changed.
|
||||
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP = 60 ; // 60 sec.
|
||||
static const uint32_t DELAY_BETWEEN_DIRECTORY_UPDATES = 600 ; // 10 minutes
|
||||
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORY_SYNC_REQ = 120 ; // 2 minutes
|
||||
static const uint32_t DELAY_BETWEEN_LOCAL_DIRECTORIES_TS_UPDATE = 20 ; // 20 sec. But we only update for real if something has changed.
|
||||
static const uint32_t DELAY_BETWEEN_REMOTE_DIRECTORIES_SWEEP = 60 ; // 60 sec.
|
||||
|
||||
static const uint32_t DELAY_BEFORE_DELETE_NON_EMPTY_REMOTE_DIR = 60*24*86400 ; // delete non empty remoe directories after 60 days of inactivity
|
||||
static const uint32_t DELAY_BEFORE_DELETE_EMPTY_REMOTE_DIR = 5*24*86400 ; // delete empty remote directories after 5 days of inactivity
|
||||
|
||||
static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time
|
||||
static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files
|
||||
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
|
||||
static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them.
|
||||
static const std::string IGNORE_DUPLICATES = "IGNORE_DUPLICATES"; // do not index files that are referenced multiple times because of links
|
||||
static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names
|
||||
static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes
|
||||
static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes
|
||||
static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags
|
||||
static const std::string MAX_SHARE_DEPTH = "MAX_SHARE_DEPTH"; // maximum depth of shared directories
|
||||
static const std::string HASH_CACHE_DURATION_SS = "HASH_CACHE_DURATION" ; // key string to store hash remembering time
|
||||
static const std::string WATCH_FILE_DURATION_SS = "WATCH_FILES_DELAY" ; // key to store delay before re-checking for new files
|
||||
static const std::string WATCH_FILE_ENABLED_SS = "WATCH_FILES_ENABLED"; // key to store ON/OFF flags for file whatch
|
||||
static const std::string TRUST_FRIEND_NODES_FOR_BANNED_FILES_SS = "TRUST_FRIEND_NODES_FOR_BANNED_FILES"; // should we trust friends for banned files or not
|
||||
static const std::string FOLLOW_SYMLINKS_SS = "FOLLOW_SYMLINKS"; // dereference symbolic links, or just ignore them.
|
||||
static const std::string IGNORE_DUPLICATES = "IGNORE_DUPLICATES"; // do not index files that are referenced multiple times because of links
|
||||
static const std::string WATCH_HASH_SALT_SS = "WATCH_HASH_SALT"; // Salt that is used to hash directory names
|
||||
static const std::string IGNORED_PREFIXES_SS = "IGNORED_PREFIXES"; // ignore file prefixes
|
||||
static const std::string IGNORED_SUFFIXES_SS = "IGNORED_SUFFIXES"; // ignore file suffixes
|
||||
static const std::string IGNORE_LIST_FLAGS_SS = "IGNORED_FLAGS"; // ignore file flags
|
||||
static const std::string MAX_SHARE_DEPTH = "MAX_SHARE_DEPTH"; // maximum depth of shared directories
|
||||
|
||||
static const std::string FILE_SHARING_DIR_NAME = "file_sharing" ; // hard-coded directory name to store friend file lists, hash cache, etc.
|
||||
static const std::string HASH_CACHE_FILE_NAME = "hash_cache.bin" ; // hard-coded directory name to store encrypted hash cache.
|
||||
|
|
@ -61,6 +62,7 @@ static const uint64_t ENTRY_INDEX_BIT_MASK_64BITS = 0x0000000
|
|||
|
||||
static const uint32_t DELAY_BEFORE_DROP_REQUEST = 600; // every 10 min
|
||||
|
||||
static const bool FOLLOW_SYMLINKS_DEFAULT = true;
|
||||
static const bool FOLLOW_SYMLINKS_DEFAULT = true;
|
||||
static const bool TRUST_FRIEND_NODES_FOR_BANNED_FILES_DEFAULT = true;
|
||||
|
||||
static const uint32_t FL_BASE_TMP_SECTION_SIZE = 4096 ;
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@
|
|||
#define P3FILELISTS_ERROR() std::cerr << "***ERROR***" << " : FILE_LISTS : " << __FUNCTION__ << " : "
|
||||
|
||||
//#define DEBUG_P3FILELISTS 1
|
||||
#define DEBUG_CONTENT_FILTERING 1
|
||||
|
||||
static const uint32_t P3FILELISTS_UPDATE_FLAG_NOTHING_CHANGED = 0x0000 ;
|
||||
static const uint32_t P3FILELISTS_UPDATE_FLAG_REMOTE_MAP_CHANGED = 0x0001 ;
|
||||
|
|
@ -72,6 +73,8 @@ p3FileDatabase::p3FileDatabase(p3ServiceControl *mpeers)
|
|||
mLastRemoteDirSweepTS = 0 ;
|
||||
mLastCleanupTime = 0 ;
|
||||
mLastDataRecvTS = 0 ;
|
||||
mTrustFriendNodesForBannedFiles = TRUST_FRIEND_NODES_FOR_BANNED_FILES_DEFAULT;
|
||||
mLastPrimaryBanListChangeTimeStamp = 0;
|
||||
|
||||
// This is for the transmission of data
|
||||
|
||||
|
|
@ -182,15 +185,19 @@ int p3FileDatabase::tick()
|
|||
|
||||
if(last_print_time + 20 < now)
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
|
||||
#ifdef DEBUG_FILE_HIERARCHY
|
||||
mLocalSharedDirs->print();
|
||||
mLocalSharedDirs->print();
|
||||
#endif
|
||||
last_print_time = now ;
|
||||
last_print_time = now ;
|
||||
}
|
||||
|
||||
#warning mr-alice 2016-08-19: "This should be removed, but it's necessary atm for updating the GUI"
|
||||
RsServer::notify()->notifyListChange(NOTIFY_LIST_DIRLIST_LOCAL, 0);
|
||||
|
||||
checkSendBannedFilesInfo();
|
||||
}
|
||||
|
||||
if(mUpdateFlags)
|
||||
|
|
@ -304,6 +311,15 @@ cleanup = true;
|
|||
}
|
||||
}
|
||||
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
|
||||
RsFileListsBannedHashesConfigItem *item = new RsFileListsBannedHashesConfigItem ;
|
||||
|
||||
item->primary_banned_files_list = mPrimaryBanList;
|
||||
sList.push_back(item) ;
|
||||
}
|
||||
|
||||
RsConfigKeyValueSet *rskv = new RsConfigKeyValueSet();
|
||||
|
||||
/* basic control parameters */
|
||||
|
|
@ -369,6 +385,14 @@ cleanup = true;
|
|||
{
|
||||
RsTlvKeyValue kv;
|
||||
|
||||
kv.key = TRUST_FRIEND_NODES_FOR_BANNED_FILES_SS;
|
||||
kv.value = trustFriendNodesForBannedFiles()?"YES":"NO" ;
|
||||
|
||||
rskv->tlvkvs.pairs.push_back(kv);
|
||||
}
|
||||
{
|
||||
RsTlvKeyValue kv;
|
||||
|
||||
kv.key = WATCH_HASH_SALT_SS;
|
||||
kv.value = mLocalDirWatcher->hashSalt().toStdString();
|
||||
|
||||
|
|
@ -462,6 +486,10 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
|
|||
{
|
||||
setWatchEnabled(kit->value == "YES") ;
|
||||
}
|
||||
else if(kit->key == TRUST_FRIEND_NODES_FOR_BANNED_FILES_SS)
|
||||
{
|
||||
setTrustFriendNodesForBannedFiles(kit->value == "YES") ;
|
||||
}
|
||||
else if(kit->key == WATCH_HASH_SALT_SS)
|
||||
{
|
||||
std::cerr << "Initing directory watcher with saved secret salt..." << std::endl;
|
||||
|
|
@ -533,6 +561,15 @@ bool p3FileDatabase::loadList(std::list<RsItem *>& load)
|
|||
dirList.push_back(info) ;
|
||||
}
|
||||
|
||||
RsFileListsBannedHashesConfigItem *fb = dynamic_cast<RsFileListsBannedHashesConfigItem*>(*it) ;
|
||||
|
||||
if(fb)
|
||||
{
|
||||
mPrimaryBanList = fb->primary_banned_files_list ;
|
||||
mBannedFileListNeedsUpdate = true;
|
||||
mLastPrimaryBanListChangeTimeStamp = time(NULL);
|
||||
}
|
||||
|
||||
delete *it ;
|
||||
}
|
||||
|
||||
|
|
@ -1402,8 +1439,8 @@ void p3FileDatabase::tickRecv()
|
|||
{
|
||||
switch(item->PacketSubType())
|
||||
{
|
||||
case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast<RsFileListsSyncRequestItem*>(item) ) ;
|
||||
break ;
|
||||
case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: handleDirSyncRequest( dynamic_cast<RsFileListsSyncRequestItem*>(item) ) ; break ;
|
||||
case RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_ITEM : handleBannedFilesInfo( dynamic_cast<RsFileListsBannedHashesItem*>(item) ) ; break ;
|
||||
case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM:
|
||||
{
|
||||
RsFileListsSyncResponseItem *sitem = dynamic_cast<RsFileListsSyncResponseItem*>(item);
|
||||
|
|
@ -1855,6 +1892,262 @@ bool p3FileDatabase::locked_generateAndSendSyncRequest(RemoteDirectoryStorage *r
|
|||
}
|
||||
|
||||
|
||||
//=========================================================================================================================//
|
||||
// Unwanted content filtering system //
|
||||
//=========================================================================================================================//
|
||||
|
||||
bool p3FileDatabase::banFile(const RsFileHash& real_file_hash, const std::string& filename, uint64_t file_size)
|
||||
{
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " setting file \"" << filename << "\" size=" << file_size << " hash=" << real_file_hash << " as banned." << std::endl;
|
||||
#endif
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
BannedFileEntry& entry(mPrimaryBanList[real_file_hash]) ; // primary list (user controlled) of files banned from FT search and forwarding. map<real hash, BannedFileEntry>
|
||||
|
||||
if(entry.ban_time_stamp == 0)
|
||||
{
|
||||
entry.filename = filename ;
|
||||
entry.size = file_size ;
|
||||
entry.ban_time_stamp = time(NULL);
|
||||
|
||||
RsFileHash hash_of_hash ;
|
||||
ftServer::encryptHash(real_file_hash,hash_of_hash) ;
|
||||
|
||||
mBannedFileList.insert(real_file_hash) ;
|
||||
mBannedFileList.insert(hash_of_hash) ;
|
||||
|
||||
mLastPrimaryBanListChangeTimeStamp = time(NULL);
|
||||
mBannedFileListNeedsUpdate = true ;
|
||||
}
|
||||
}
|
||||
|
||||
IndicateConfigChanged();
|
||||
return true;
|
||||
}
|
||||
bool p3FileDatabase::unbanFile(const RsFileHash& real_file_hash)
|
||||
{
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " unbanning file with hash " << real_file_hash << std::endl;
|
||||
#endif
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
mPrimaryBanList.erase(real_file_hash) ;
|
||||
mLastPrimaryBanListChangeTimeStamp = time(NULL);
|
||||
mBannedFileListNeedsUpdate = true ;
|
||||
}
|
||||
|
||||
IndicateConfigChanged();
|
||||
return true;
|
||||
}
|
||||
|
||||
bool p3FileDatabase::isFileBanned(const RsFileHash& hash)
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
|
||||
if(mBannedFileList.empty()) // quick exit
|
||||
return false ;
|
||||
|
||||
RsFileHash hash_of_hash ;
|
||||
ftServer::encryptHash(hash,hash_of_hash) ;
|
||||
|
||||
bool res = mBannedFileList.find(hash) != mBannedFileList.end() || mBannedFileList.find(hash_of_hash) != mBannedFileList.end() ;
|
||||
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
if(res)
|
||||
P3FILELISTS_DEBUG() << " is file banned(" << hash << "): " << (res?"YES":"NO") << std::endl;
|
||||
#endif
|
||||
return res ;
|
||||
}
|
||||
|
||||
bool p3FileDatabase::getPrimaryBannedFilesList(std::map<RsFileHash,BannedFileEntry>& banned_files)
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
banned_files = mPrimaryBanList;
|
||||
|
||||
return true ;
|
||||
}
|
||||
|
||||
bool p3FileDatabase::trustFriendNodesForBannedFiles() const
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
return mTrustFriendNodesForBannedFiles;
|
||||
}
|
||||
|
||||
void p3FileDatabase::setTrustFriendNodesForBannedFiles(bool b)
|
||||
{
|
||||
if(b != mTrustFriendNodesForBannedFiles)
|
||||
{
|
||||
IndicateConfigChanged();
|
||||
mBannedFileListNeedsUpdate = true;
|
||||
}
|
||||
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
mTrustFriendNodesForBannedFiles = b;
|
||||
}
|
||||
|
||||
void p3FileDatabase::checkSendBannedFilesInfo()
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
|
||||
// 1 - compare records to list of online friends, send own info of not already
|
||||
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " Checking banned files information: " << std::endl;
|
||||
#endif
|
||||
|
||||
time_t now = time(NULL);
|
||||
std::list<RsPeerId> online_friends ;
|
||||
rsPeers->getOnlineList(online_friends);
|
||||
|
||||
std::set<RsPeerId> peers ;
|
||||
for(auto it(online_friends.begin());it!=online_friends.end();++it) // convert to std::set for efficient search
|
||||
peers.insert(*it) ;
|
||||
|
||||
for(auto it(mPeerBannedFiles.begin());it!=mPeerBannedFiles.end();)
|
||||
{
|
||||
if(peers.find(it->first) == peers.end()) // friend not online, remove his record
|
||||
{
|
||||
it = mPeerBannedFiles.erase(it) ;
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " Peer " << it->first << " is offline: removign record." << std::endl;
|
||||
#endif
|
||||
continue;
|
||||
}
|
||||
|
||||
if(it->second.mLastSent < mLastPrimaryBanListChangeTimeStamp) // has ban info already been sent? If not do it.
|
||||
{
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " Peer " << it->first << " is online and hasn't been sent since last change: sending..." << std::endl;
|
||||
#endif
|
||||
locked_sendBanInfo(it->first);
|
||||
it->second.mLastSent = now;
|
||||
}
|
||||
|
||||
peers.erase(it->first); // friend has been handled -> remove from list
|
||||
++it;
|
||||
}
|
||||
|
||||
// 2 - add a new record for friends not already in the record map
|
||||
|
||||
for(auto it(peers.begin());it!=peers.end();++it)
|
||||
{
|
||||
locked_sendBanInfo(*it);
|
||||
mPeerBannedFiles[*it].mLastSent = now;
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " Peer " << *it << " is online and hasn't been sent info at all: sending..." << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
// 3 - update list of banned hashes if it has changed somehow
|
||||
|
||||
if(mBannedFileListNeedsUpdate)
|
||||
{
|
||||
mBannedFileList.clear();
|
||||
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " Creating local banned file list: " << std::endl;
|
||||
#endif
|
||||
// Add all H(H(f)) from friends
|
||||
|
||||
if(mTrustFriendNodesForBannedFiles)
|
||||
for(auto it(mPeerBannedFiles.begin());it!=mPeerBannedFiles.end();++it)
|
||||
for(auto it2(it->second.mBannedHashOfHash.begin());it2!=it->second.mBannedHashOfHash.end();++it2)
|
||||
{
|
||||
mBannedFileList.insert(*it2);
|
||||
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " from " << it->first << ": H(H(f)) = " << *it2 << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
// Add H(f) and H(H(f)) from our primary list
|
||||
|
||||
for(auto it(mPrimaryBanList.begin());it!=mPrimaryBanList.end();++it)
|
||||
{
|
||||
mBannedFileList.insert(it->first) ;
|
||||
|
||||
RsFileHash hash_of_hash ;
|
||||
ftServer::encryptHash(it->first,hash_of_hash) ;
|
||||
|
||||
mBannedFileList.insert(hash_of_hash) ;
|
||||
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " primary: H(f) = " << it->first << ": H(H(f)) = " << hash_of_hash << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
mBannedFileListNeedsUpdate = false ;
|
||||
}
|
||||
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " Final list of locally banned hashes contains: " << mBannedFileList.size() << " elements." << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
void p3FileDatabase::locked_sendBanInfo(const RsPeerId& peer)
|
||||
{
|
||||
RsFileListsBannedHashesItem *item = new RsFileListsBannedHashesItem ;
|
||||
uint32_t session_id = RSRandom::random_u32();
|
||||
|
||||
item->session_id = session_id ;
|
||||
item->PeerId(peer);
|
||||
|
||||
for(auto it(mPrimaryBanList.begin());it!=mPrimaryBanList.end();++it)
|
||||
{
|
||||
RsFileHash hash_of_hash ;
|
||||
|
||||
ftServer::encryptHash(it->first,hash_of_hash) ;
|
||||
|
||||
if(!item)
|
||||
{
|
||||
item = new RsFileListsBannedHashesItem ;
|
||||
|
||||
item->PeerId(peer);
|
||||
item->session_id = session_id ;
|
||||
}
|
||||
|
||||
item->encrypted_hashes.insert(hash_of_hash) ;
|
||||
|
||||
if(item->encrypted_hashes.size() >= 200)
|
||||
{
|
||||
sendItem(item);
|
||||
item = NULL ;
|
||||
}
|
||||
}
|
||||
|
||||
if(item)
|
||||
sendItem(item);
|
||||
}
|
||||
|
||||
void p3FileDatabase::handleBannedFilesInfo(RsFileListsBannedHashesItem *item)
|
||||
{
|
||||
RS_STACK_MUTEX(mFLSMtx) ;
|
||||
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " received banned files info from peer " << item->PeerId() << ", session id = " << std::hex << item->session_id << std::dec << ": " << item->encrypted_hashes.size() << " files:" << std::endl;
|
||||
#endif
|
||||
// 1 - localize the friend in the banned file map
|
||||
|
||||
PeerBannedFilesEntry& pbfe(mPeerBannedFiles[item->PeerId()]) ;
|
||||
|
||||
if(pbfe.mSessionId != item->session_id)
|
||||
pbfe.mBannedHashOfHash.clear();
|
||||
|
||||
pbfe.mSessionId = item->session_id ;
|
||||
|
||||
// 2 - replace/update the list, depending on the session_id
|
||||
|
||||
for(auto it(item->encrypted_hashes.begin());it!=item->encrypted_hashes.end();++it)
|
||||
{
|
||||
pbfe.mBannedHashOfHash.insert(*it);
|
||||
#ifdef DEBUG_CONTENT_FILTERING
|
||||
P3FILELISTS_DEBUG() << " added H(H(f)) = " << *it << std::endl;
|
||||
#endif
|
||||
}
|
||||
|
||||
// 3 - tell the updater that the banned file list has changed
|
||||
|
||||
mBannedFileListNeedsUpdate = true ;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -62,9 +62,17 @@ class LocalDirectoryStorage ;
|
|||
|
||||
class RsFileListsSyncRequestItem ;
|
||||
class RsFileListsSyncResponseItem ;
|
||||
class RsFileListsBannedHashesItem ;
|
||||
|
||||
class HashStorage ;
|
||||
|
||||
struct PeerBannedFilesEntry
|
||||
{
|
||||
std::set<RsFileHash> mBannedHashOfHash;
|
||||
uint32_t mSessionId ; // used for when a friend sends multiple packets in separate items.
|
||||
time_t mLastSent;
|
||||
};
|
||||
|
||||
class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, public RsSharedFileService
|
||||
{
|
||||
public:
|
||||
|
|
@ -132,6 +140,13 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
|
|||
void setMaxShareDepth(int i) ;
|
||||
int maxShareDepth() const ;
|
||||
|
||||
bool banFile(const RsFileHash& real_file_hash, const std::string& filename, uint64_t file_size) ;
|
||||
bool unbanFile(const RsFileHash& real_file_hash);
|
||||
bool isFileBanned(const RsFileHash& hash) ;
|
||||
bool getPrimaryBannedFilesList(std::map<RsFileHash,BannedFileEntry>& banned_files) ;
|
||||
bool trustFriendNodesForBannedFiles() const ;
|
||||
void setTrustFriendNodesForBannedFiles(bool b) ;
|
||||
|
||||
// computes/gathers statistics about shared directories
|
||||
|
||||
int getSharedDirStatistics(const RsPeerId& pid,SharedDirStats& stats);
|
||||
|
|
@ -168,6 +183,8 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
|
|||
void tickRecv();
|
||||
void tickSend();
|
||||
|
||||
void checkSendBannedFilesInfo();
|
||||
|
||||
private:
|
||||
p3ServiceControl *mServCtrl ;
|
||||
RsPeerId mOwnId ;
|
||||
|
|
@ -244,5 +261,18 @@ class p3FileDatabase: public p3Service, public p3Config, public ftSearch //, pub
|
|||
std::string mFileSharingDir ;
|
||||
time_t mLastCleanupTime;
|
||||
time_t mLastDataRecvTS ;
|
||||
|
||||
// File filtering. Not explicitly related to shared files, but has its place here
|
||||
//
|
||||
|
||||
std::map<RsFileHash,BannedFileEntry> mPrimaryBanList ; // primary list (user controlled) of files banned from FT search and forwarding. map<real hash, BannedFileEntry>
|
||||
std::map<RsPeerId,PeerBannedFilesEntry> mPeerBannedFiles ; // records of which files other peers ban, stored as H(H(f))
|
||||
std::set<RsFileHash> mBannedFileList ; // list of banned hashes. This include original hashs and H(H(f)) when coming from friends.
|
||||
bool mTrustFriendNodesForBannedFiles ;
|
||||
bool mBannedFileListNeedsUpdate;
|
||||
time_t mLastPrimaryBanListChangeTimeStamp;
|
||||
|
||||
void locked_sendBanInfo(const RsPeerId& pid);
|
||||
void handleBannedFilesInfo(RsFileListsBannedHashesItem *item);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -34,14 +34,28 @@ void RsFileListsSyncRequestItem::serial_process(RsGenericSerializer::SerializeJo
|
|||
}
|
||||
void RsFileListsSyncResponseItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
|
||||
{
|
||||
RsTypeSerializer::serial_process (j,ctx,entry_hash,"entry_hash") ;
|
||||
RsTypeSerializer::serial_process (j,ctx,checksum,"checksum") ;
|
||||
RsTypeSerializer::serial_process<uint32_t> (j,ctx,flags ,"flags") ;
|
||||
RsTypeSerializer::serial_process (j,ctx,entry_hash, "entry_hash") ;
|
||||
RsTypeSerializer::serial_process (j,ctx,checksum, "checksum") ;
|
||||
RsTypeSerializer::serial_process<uint32_t> (j,ctx,flags, "flags") ;
|
||||
RsTypeSerializer::serial_process<uint32_t> (j,ctx,last_known_recurs_modf_TS,"last_known_recurs_modf_TS") ;
|
||||
RsTypeSerializer::serial_process<uint64_t> (j,ctx,request_id,"request_id") ;
|
||||
RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,directory_content_data,"directory_content_data") ;
|
||||
RsTypeSerializer::serial_process<uint64_t> (j,ctx,request_id, "request_id") ;
|
||||
RsTypeSerializer::serial_process<RsTlvItem>(j,ctx,directory_content_data, "directory_content_data") ;
|
||||
}
|
||||
void RsFileListsBannedHashesItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
|
||||
{
|
||||
RsTypeSerializer::serial_process(j,ctx,session_id ,"session_id") ;
|
||||
RsTypeSerializer::serial_process(j,ctx,encrypted_hashes,"encrypted_hashes") ;
|
||||
}
|
||||
void RsFileListsBannedHashesConfigItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
|
||||
{
|
||||
RsTypeSerializer::serial_process(j,ctx,primary_banned_files_list,"primary_banned_files_list") ;
|
||||
}
|
||||
template<> void RsTypeSerializer::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx,BannedFileEntry& entry,const std::string& /*name*/)
|
||||
{
|
||||
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_NAME,entry.filename ,"entry.file_name") ;
|
||||
RsTypeSerializer::serial_process<uint64_t>(j,ctx, entry.size ,"entry.size") ;
|
||||
RsTypeSerializer::serial_process<time_t> (j,ctx, entry.ban_time_stamp,"entry.ban_time_stamp") ;
|
||||
}
|
||||
|
||||
RsItem *RsFileListsSerialiser::create_item(uint16_t service,uint8_t type) const
|
||||
{
|
||||
if(service != RS_SERVICE_TYPE_FILE_DATABASE)
|
||||
|
|
@ -49,8 +63,10 @@ RsItem *RsFileListsSerialiser::create_item(uint16_t service,uint8_t type) const
|
|||
|
||||
switch(type)
|
||||
{
|
||||
case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return new RsFileListsSyncRequestItem();
|
||||
case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: return new RsFileListsSyncResponseItem();
|
||||
case RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM: return new RsFileListsSyncRequestItem();
|
||||
case RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM: return new RsFileListsSyncResponseItem();
|
||||
case RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_ITEM: return new RsFileListsBannedHashesItem();
|
||||
case RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_CONFIG_ITEM: return new RsFileListsBannedHashesConfigItem();
|
||||
default:
|
||||
return NULL ;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -33,12 +33,13 @@
|
|||
#include "gxs/rsgxsdata.h"
|
||||
|
||||
#include "serialiser/rsserializer.h"
|
||||
#include "retroshare/rsfiles.h"
|
||||
|
||||
// These items have "flag type" numbers, but this is not used.
|
||||
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01;
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM = 0x02;
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03;
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_REQ_ITEM = 0x01;
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_SYNC_RSP_ITEM = 0x02;
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_CONFIG_ITEM = 0x03;
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_ITEM = 0x04;
|
||||
const uint8_t RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_CONFIG_ITEM = 0x05;
|
||||
|
||||
/*!
|
||||
* Base class for filelist sync items
|
||||
|
|
@ -103,6 +104,29 @@ public:
|
|||
RsTlvBinaryData directory_content_data ; // encoded binary data. This allows to vary the encoding format, in a way that is transparent to the serialiser.
|
||||
};
|
||||
|
||||
class RsFileListsBannedHashesItem: public RsFileListsItem
|
||||
{
|
||||
public:
|
||||
RsFileListsBannedHashesItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_ITEM){}
|
||||
|
||||
virtual void clear() { encrypted_hashes.clear(); }
|
||||
virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
|
||||
|
||||
uint32_t session_id ; // used to allow to send in multiple parts.
|
||||
std::set<RsFileHash> encrypted_hashes ;// hash of hash for each banned file.
|
||||
};
|
||||
|
||||
class RsFileListsBannedHashesConfigItem: public RsFileListsItem
|
||||
{
|
||||
public:
|
||||
RsFileListsBannedHashesConfigItem() : RsFileListsItem(RS_PKT_SUBTYPE_FILELISTS_BANNED_HASHES_CONFIG_ITEM){}
|
||||
|
||||
virtual void clear() { primary_banned_files_list.clear(); }
|
||||
virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
|
||||
|
||||
std::map<RsFileHash,BannedFileEntry> primary_banned_files_list ;
|
||||
};
|
||||
|
||||
class RsFileListsSerialiser : public RsServiceSerializer
|
||||
{
|
||||
public:
|
||||
|
|
|
|||
|
|
@ -1837,8 +1837,7 @@ void ftServer::receiveSearchResult(RsTurtleFTSearchResultItem *item)
|
|||
} // end RS_STACK_MUTEX(mSearchCallbacksMapMutex);
|
||||
|
||||
if(!hasCallback)
|
||||
RsServer::notify()->notifyTurtleSearchResult(
|
||||
item->request_id, item->result );
|
||||
RsServer::notify()->notifyTurtleSearchResult(item->PeerId(),item->request_id, item->result );
|
||||
}
|
||||
|
||||
/***************************** CONFIG ****************************/
|
||||
|
|
@ -1846,8 +1845,8 @@ void ftServer::receiveSearchResult(RsTurtleFTSearchResultItem *item)
|
|||
bool ftServer::addConfiguration(p3ConfigMgr *cfgmgr)
|
||||
{
|
||||
/* add all the subbits to config mgr */
|
||||
cfgmgr->addConfiguration("ft_database.cfg", mFileDatabase);
|
||||
cfgmgr->addConfiguration("ft_extra.cfg", mFtExtra);
|
||||
cfgmgr->addConfiguration("ft_database.cfg" , mFileDatabase);
|
||||
cfgmgr->addConfiguration("ft_extra.cfg" , mFtExtra );
|
||||
cfgmgr->addConfiguration("ft_transfers.cfg", mFtController);
|
||||
|
||||
return true;
|
||||
|
|
@ -1888,3 +1887,25 @@ void ftServer::cleanTimedOutSearches()
|
|||
cbpt = mSearchCallbacksMap.erase(cbpt);
|
||||
else ++cbpt;
|
||||
}
|
||||
|
||||
// Offensive content file filtering
|
||||
|
||||
int ftServer::banFile(const RsFileHash& real_file_hash, const std::string& filename, uint64_t file_size)
|
||||
{
|
||||
return mFileDatabase->banFile(real_file_hash,filename,file_size) ;
|
||||
}
|
||||
int ftServer::unbanFile(const RsFileHash& real_file_hash)
|
||||
{
|
||||
return mFileDatabase->unbanFile(real_file_hash) ;
|
||||
}
|
||||
bool ftServer::getPrimaryBannedFilesList(std::map<RsFileHash,BannedFileEntry>& banned_files)
|
||||
{
|
||||
return mFileDatabase->getPrimaryBannedFilesList(banned_files) ;
|
||||
}
|
||||
|
||||
bool ftServer::isHashBanned(const RsFileHash& hash)
|
||||
{
|
||||
return mFileDatabase->isFileBanned(hash);
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -200,6 +200,11 @@ public:
|
|||
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id);
|
||||
virtual int getSharedDirStatistics(const RsPeerId& pid, SharedDirStats& stats) ;
|
||||
|
||||
virtual int banFile(const RsFileHash& real_file_hash, const std::string& filename, uint64_t file_size) ;
|
||||
virtual int unbanFile(const RsFileHash& real_file_hash);
|
||||
virtual bool getPrimaryBannedFilesList(std::map<RsFileHash,BannedFileEntry>& banned_files) ;
|
||||
virtual bool isHashBanned(const RsFileHash& hash);
|
||||
|
||||
/***
|
||||
* Utility Functions
|
||||
***/
|
||||
|
|
@ -245,6 +250,8 @@ public:
|
|||
virtual bool ignoreDuplicates() ;
|
||||
virtual void setIgnoreDuplicates(bool ignore) ;
|
||||
|
||||
static bool encryptHash(const RsFileHash& hash, RsFileHash& hash_of_hash);
|
||||
|
||||
/***************************************************************/
|
||||
/*************** Data Transfer Interface ***********************/
|
||||
/***************************************************************/
|
||||
|
|
@ -290,7 +297,6 @@ protected:
|
|||
// fnds out what is the real hash of encrypted hash hash
|
||||
bool findRealHash(const RsFileHash& hash, RsFileHash& real_hash);
|
||||
bool findEncryptedHash(const RsPeerId& virtual_peer_id, RsFileHash& encrypted_hash);
|
||||
bool encryptHash(const RsFileHash& hash, RsFileHash& hash_of_hash);
|
||||
|
||||
bool checkUploadLimit(const RsPeerId& pid,const RsFileHash& hash);
|
||||
private:
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@
|
|||
#include "gxs/rsnxs.h"
|
||||
#include "rsgxsnettunnel.h"
|
||||
|
||||
#define DEBUG_RSGXSNETTUNNEL 1
|
||||
// #define DEBUG_RSGXSNETTUNNEL 1
|
||||
|
||||
#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; }
|
||||
#define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL: " << __FUNCTION__ << " : "
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ void p3Notify::notifyChatCleared (const ChatId& chat_id)
|
|||
void p3Notify::notifyChatLobbyTimeShift (int time_shift) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyChatLobbyTimeShift(time_shift) ; }
|
||||
void p3Notify::notifyCustomState (const std::string& peer_id , const std::string& status_string ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyCustomState (peer_id,status_string) ; }
|
||||
void p3Notify::notifyHashingInfo (uint32_t type , const std::string& fileinfo ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyHashingInfo (type,fileinfo) ; }
|
||||
void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list<TurtleFileInfo>& files ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,files) ; }
|
||||
void p3Notify::notifyTurtleSearchResult (const RsPeerId& pid , uint32_t search_id , const std::list<TurtleFileInfo>& files ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(pid,search_id,files) ; }
|
||||
#warning MISSING CODE HERE
|
||||
//void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list<TurtleGxsInfo>& groups ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; }
|
||||
void p3Notify::notifyPeerHasNewAvatar (std::string peer_id ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; }
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ class p3Notify: public RsNotify
|
|||
void notifyChatLobbyTimeShift (int /* time_shift*/) ;
|
||||
void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) ;
|
||||
void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) ;
|
||||
void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleFileInfo>& /* files */) ;
|
||||
void notifyTurtleSearchResult (const RsPeerId &pid, uint32_t /* search_id */, const std::list<TurtleFileInfo>& /* files */) ;
|
||||
#warning MISSING CODE HERE
|
||||
// void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleGxsInfo >& /* groups */) ;
|
||||
void notifyPeerHasNewAvatar (std::string /* peer_id */) ;
|
||||
|
|
|
|||
|
|
@ -181,6 +181,15 @@ public:
|
|||
uint64_t mTotalSize ;
|
||||
};
|
||||
|
||||
struct BannedFileEntry
|
||||
{
|
||||
BannedFileEntry() : size(0),filename(""),ban_time_stamp(0) {}
|
||||
|
||||
uint64_t size ;
|
||||
std::string filename ;
|
||||
time_t ban_time_stamp;
|
||||
};
|
||||
|
||||
class RsFiles
|
||||
{
|
||||
public:
|
||||
|
|
@ -421,6 +430,11 @@ public:
|
|||
virtual int SearchBoolExp(RsRegularExpression::Expression * exp, std::list<DirDetails> &results,FileSearchFlags flags,const RsPeerId& peer_id) = 0;
|
||||
virtual int getSharedDirStatistics(const RsPeerId& pid, SharedDirStats& stats) =0;
|
||||
|
||||
virtual int banFile(const RsFileHash& real_file_hash, const std::string& filename, uint64_t file_size) =0;
|
||||
virtual int unbanFile(const RsFileHash& real_file_hash)=0;
|
||||
virtual bool getPrimaryBannedFilesList(std::map<RsFileHash,BannedFileEntry>& banned_files) =0;
|
||||
virtual bool isHashBanned(const RsFileHash& hash) =0;
|
||||
|
||||
/***
|
||||
* Utility Functions.
|
||||
***/
|
||||
|
|
|
|||
|
|
@ -222,7 +222,7 @@ public:
|
|||
virtual void notifyChatLobbyTimeShift (int /* time_shift*/) {}
|
||||
virtual void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) {}
|
||||
virtual void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) {}
|
||||
virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleFileInfo>& /* files */) {}
|
||||
virtual void notifyTurtleSearchResult (const RsPeerId& /* pid */, uint32_t /* search_id */, const std::list<TurtleFileInfo>& /* files */) {}
|
||||
#warning MISSING CODE HERE
|
||||
// virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleGxsInfo >& /* groups */) {}
|
||||
virtual void notifyPeerHasNewAvatar (std::string /* peer_id */) {}
|
||||
|
|
|
|||
|
|
@ -1108,6 +1108,27 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint
|
|||
|
||||
void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
|
||||
{
|
||||
// Filter out banned hashes from the result.
|
||||
|
||||
RsTurtleFTSearchResultItem *ftsr_tmp = dynamic_cast<RsTurtleFTSearchResultItem*>(item) ;
|
||||
|
||||
if(ftsr_tmp != NULL)
|
||||
{
|
||||
for(auto it(ftsr_tmp->result.begin());it!=ftsr_tmp->result.end();)
|
||||
if( rsFiles->isHashBanned((*it).hash) )
|
||||
{
|
||||
std::cerr << "(II) filtering out banned hash " << (*it).hash << " from turtle result " << std::hex << item->request_id << std::dec << std::endl;
|
||||
it = ftsr_tmp->result.erase(it);
|
||||
}
|
||||
else
|
||||
++it;
|
||||
|
||||
if(ftsr_tmp->result.empty())
|
||||
return ;
|
||||
}
|
||||
|
||||
// Then handle the result
|
||||
|
||||
std::list<std::pair<RsTurtleSearchResultItem*,RsTurtleClientService*> > results_to_notify_off_mutex ;
|
||||
|
||||
{
|
||||
|
|
@ -1532,6 +1553,14 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
|
|||
item->print(std::cerr,0) ;
|
||||
#endif
|
||||
|
||||
// check first if the hash is in the ban list. If so, drop the request.
|
||||
|
||||
if(rsFiles->isHashBanned(item->file_hash))
|
||||
{
|
||||
std::cerr << "(II) Rejecting tunnel request to ban hash " << item->file_hash << std::endl;
|
||||
return ;
|
||||
}
|
||||
|
||||
#ifdef TUNNEL_STATISTICS
|
||||
if(TS_request_bounces.find(item->request_id) != TS_request_bounces.end())
|
||||
TS_request_bounces[item->request_id].push_back(time(NULL)) ;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue