Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-01-13 08:29:32 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merging from branches/v0.5-peernet/libbitdht (--- Merging r4354 through r4356 into '.')

Browse Source 
* adding Bloom filter to speed up finding RS peers.
 * fixed startup bug. (loops into FAILURE)




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@4357 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
drbob 2011-06-29 16:06:33 +00:00
parent 702ff44194
commit 79b48d5a89
15 changed files with 820 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
libbitdht/src/bitdht/bdaccount.cc
View File

@ -64,7 +64,7 @@ bdAccount::bdAccount()
void bdAccount::incCounter(uint32_t idx, bool out)
{
if ((idx < 0) || (idx > mNoStats-1))
if (idx > mNoStats-1)
{
std::cerr << "bdAccount::incCounter() Invalid Index";
std::cerr << std::endl;

55
libbitdht/src/bitdht/bdmanager.cc
View File

@ -74,6 +74,10 @@ bdNodeManager::bdNodeManager(bdNodeId *id, std::string dhtVersion, std::string b
mNetworkSize = 0;
mBdNetworkSize = 0;
std::string bfilter = "15108044002028000000200040400081a20840001000010400070240220000020501196b2420105218204008000000201102802041000a000004000004002881001180068301458000180000040000010080820e0005811000220200040800210280582001118024041002200004000000c44400080485a50008011084040200";
mBloomFilter.setFilterBits(bfilter);
#ifdef DEBUG_MGR
std::cerr << "bdNodeManager::bdNodeManager() ID: ";
mFns->bdPrintNodeId(std::cerr, id);
@ -379,11 +383,19 @@ void bdNodeManager::iteration()
status(); /* calculates mNetworkSize */
#ifdef DEBUG_MGR
mAccount.printStats(std::cerr);
#endif
/* Finally, Fail, and restart if we lose all peers */
if (mNetworkSize < MIN_OP_SPACE_SIZE)
uint32_t nodeSpaceSize = mNodeSpace.calcSpaceSize();
if (nodeSpaceSize < MIN_OP_SPACE_SIZE)
{
std::cerr << "bdNodeManager::iteration(): SpaceSize to Small: " << nodeSpaceSize;
std::cerr << std::endl;
std::cerr << "bdNodeManager::iteration(): REFRESH ==> FAILED";
std::cerr << std::endl;
mMode = BITDHT_MGR_STATE_FAILED;
mModeTS = now;
}
@ -459,28 +471,30 @@ int bdNodeManager::QueryRandomLocalNet()
mQueryMgr->addWorthyPeerSource(&id); /* Tell BitDHT that we really want to ping their peers */
send_query(&id, &targetNodeId);
#ifdef DEBUG_NODE_MSGS
std::cerr << "bdNodeManager::QueryRandomLocalNet() Querying : ";
mFns->bdPrintId(std::cerr, &id);
std::cerr << " searching for : ";
mFns->bdPrintNodeId(std::cerr, &targetNodeId);
std::cerr << std::endl;
#endif
if (isRandom)
{
#ifdef DEBUG_NODE_MSGS
std::cerr << "bdNodeManager::QueryRandomLocalNet() Search is Random!";
std::cerr << std::endl;
#endif
}
#ifdef DEBUG_NODE_MSGS
#endif
return 1;
}
else
{
#ifdef DEBUG_NODE_MSGS
#endif
std::cerr << "bdNodeManager::QueryRandomLocalNet() No LocalNet Peer Found";
std::cerr << std::endl;
#endif
}
return 0;
@ -525,7 +539,38 @@ void bdNodeManager::SearchForLocalNet()
{
/* install a new query */
bdNodeId targetNodeId;
bdStdRandomNodeId(&targetNodeId);
/* get something that filter approves of */
bool filterOk = false;
int i;
#define MAX_FILTER_ATTEMPTS 3000
for(i = 0; (!filterOk) && (i < MAX_FILTER_ATTEMPTS); i++)
{
bdStdRandomNodeId(&targetNodeId);
std::ostringstream tststr;
bdStdPrintNodeId(tststr, &targetNodeId);
if (mBloomFilter.test(tststr.str()))
{
filterOk = true;
}
}
if (filterOk)
{
std::cerr << "bdNodeManager::SearchForLocalNet() " << i << " Attempts to find OkNode: ";
mFns->bdPrintNodeId(std::cerr, &targetNodeId);
std::cerr << std::endl;
}
else
{
std::cerr << "bdNodeManager::SearchForLocalNet() Failed to Find FilterOk this time: ";
mFns->bdPrintNodeId(std::cerr, &targetNodeId);
std::cerr << std::endl;
}
uint32_t qflags = BITDHT_QFLAGS_INTERNAL | BITDHT_QFLAGS_DISGUISE;
mQueryMgr->addQuery(&targetNodeId, qflags);

7
libbitdht/src/bitdht/bdmanager.h
View File

@ -53,8 +53,9 @@
#define BITDHT_PS_STATE_ONLINE (0x00000400)
#define BITDHT_PS_STATE_CONNECTED (0x00000800)
#include "bdiface.h"
#include "bdnode.h"
#include "bitdht/bdiface.h"
#include "bitdht/bdnode.h"
#include "util/bdbloom.h"
@ -171,6 +172,8 @@ void SearchForLocalNet();
uint32_t mNetworkSize;
uint32_t mBdNetworkSize;
bdBloom mBloomFilter;
/* future node functions */
//addPeerPing(foundId);
//clearPing(it->first);

12
libbitdht/src/bitdht/bdnode.cc
View File

@ -461,6 +461,18 @@ void bdNode::addPeer(const bdId *id, uint32_t peerflags)
// Finally we pass to connections for them to use.
mConnMgr->updatePotentialConnectionProxy(id, peerflags);
//#define DISPLAY_BITDHTNODES 1
#ifdef DISPLAY_BITDHTNODES
/* TEMP to extract IDS for BloomFilter */
if (peerflags & BITDHT_PEER_STATUS_DHT_ENGINE)
{
std::cerr << "bdNode::addPeer() FOUND BITDHT PEER";
std::cerr << std::endl;
mFns->bdPrintNodeId(std::cerr, &(id->id));
std::cerr << std::endl;
}
#endif
}
/************************************ Process Remote Query *************************/

5
libbitdht/src/bitdht/bdpeer.cc
View File

@ -669,6 +669,7 @@ int bdSpace::updateAttachedPeers()
}
}
}
return 1;
}
@ -1220,8 +1221,8 @@ bool bdSpace::findRandomPeerWithFlag(bdId &id, uint32_t withFlag)
uint32_t i = 0;
uint32_t buck = 0;
std::cerr << "bdSpace::findRandomPeerWithFlag()";
std::cerr << std::endl;
//std::cerr << "bdSpace::findRandomPeerWithFlag()";
//std::cerr << std::endl;
it = buckets.begin();
if (it != buckets.end())

13
libbitdht/src/bitdht/bdquerymgr.cc
View File

@ -297,6 +297,7 @@ int bdQueryManager::getResults(bdNodeId *target, std::list<bdId> &answer, int q
/* will only be one matching query.. so end loop */
return results;
}
return 0;
}
@ -328,10 +329,10 @@ void bdQueryManager::addWorthyPeerSource(bdId *src)
peer.mPeerId = *src;
peer.mFoundTime = now;
#ifdef DEBUG_NODE_ACTIONS
std::cerr << "bdQueryManager::addWorthyPeerSource(";
mFns->bdPrintId(std::cerr, src);
std::cerr << ")" << std::endl;
#ifdef DEBUG_NODE_ACTIONS
#endif
mWorthyPeerSources.push_back(peer);
@ -348,9 +349,11 @@ bool bdQueryManager::checkWorthyPeerSources(bdId *src)
{
if (now - it->mFoundTime > MAX_WORTHY_PEER_AGE)
{
#ifdef DEBUG_NODE_ACTIONS
std::cerr << "bdQueryManager::checkWorthyPeerSource() Discard old Source: ";
mFns->bdPrintId(std::cerr, &(it->mPeerId));
std::cerr << std::endl;
#endif
it = mWorthyPeerSources.erase(it);
}
@ -358,9 +361,11 @@ bool bdQueryManager::checkWorthyPeerSources(bdId *src)
{
if (it->mPeerId == *src)
{
//std::cerr << "bdQueryManager::checkWorthyPeerSource(";
//mFns->bdPrintId(std::cerr, src);
//std::cerr << ") = true" << std::endl;
#ifdef DEBUG_NODE_ACTIONS
std::cerr << "bdQueryManager::checkWorthyPeerSource(";
mFns->bdPrintId(std::cerr, src);
std::cerr << ") = true" << std::endl;
#endif
return true;
}

1
libbitdht/src/bitdht/bdstore.cc
View File

@ -144,6 +144,7 @@ int bdStore::filterIpList(const std::list<struct sockaddr_in> &filteredIPs)
}
}
}
return 1;
}

2
libbitdht/src/libbitdht.pro
View File

@ -108,6 +108,7 @@ HEADERS += \
bitdht/bdfilter.h \
bitdht/bdaccount.h \
bitdht/bdquerymgr.h \
util/bdbloom.h \
SOURCES += \
bitdht/bencode.c \
@ -131,5 +132,6 @@ SOURCES += \
bitdht/bdfilter.cc \
bitdht/bdaccount.cc \
bitdht/bdquerymgr.cc \
util/bdbloom.cc \

21
libbitdht/src/tests/Makefile
View File

@ -11,14 +11,19 @@ include $(TEST_TOP_DIR)/scripts/config.mk
# Generic Test Harnesses.
TESTOBJ = bdmetric_test.o bdmsgs_test.o bdnode_test.o bdspace_test.o
TESTOBJ += bdmgr_multitest.o bdnode_multitest1.o bdquery_test.o bdstore_test.o
TESTOBJ += bdmidids_test.o bdnode_test2.o bdspace_test2.o udpbitdht_nettest.o
TESTOBJ += bdmgr_multitest.o bdquery_test.o bdstore_test.o
TESTOBJ += bdmidids_test.o bdspace_test2.o udpbitdht_nettest.o
TESTOBJ += bdbloom_test.o bdbloom_makefilter.o
#TESTOBJ += bencode_test.o bdudp_test.o
#TESTOBJ = bdnode_test.o bdnode_multitest1.o bdnode_test2.o
TESTS = bdmetric_test bdmsgs_test bdnode_test bdspace_test
TESTS += bdmgr_multitest bdnode_multitest1 bdquery_test bdstore_test
TESTS += bdmidids_test bdnode_test2 bdspace_test2 udpbitdht_nettest
TESTS = bdmetric_test bdmsgs_test bdspace_test
TESTS += bdmgr_multitest bdquery_test bdstore_test
TESTS += bdmidids_test bdspace_test2 udpbitdht_nettest
TESTS += bdbloom_test
#TESTS += bencode_test bdudp_test
#Tests to Fix.
#TESTS = bdnode_test bdnode_multitest1 bdnode_test2
MANUAL_TESTS =
@ -68,6 +73,12 @@ udpbitdht_nettest: udpbitdht_nettest.o
bencode_test: bencode_test.o
$(CC) $(CFLAGS) -o bencode_test bencode_test.o $(LIBS)
bdbloom_test: bdbloom_test.o
$(CC) $(CFLAGS) -o bdbloom_test bdbloom_test.o $(LIBS)
bdbloom_makefilter: bdbloom_makefilter.o
$(CC) $(CFLAGS) -o bdbloom_makefilter bdbloom_makefilter.o $(LIBS)
clobber: remove_extra_files

91
libbitdht/src/tests/bdbloom_makefilter.cc Normal file
View File

@ -0,0 +1,91 @@
/*
* bitdht/bdbloom_test.cc
*
* BitDHT: An Flexible DHT library.
*
* Copyright 2010 by Robert Fernie
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 3 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "bitdht@lunamutt.com".
*
*/
#include "util/bdbloom.h"
#include "bitdht/bdstddht.h"
#include <iostream>
#include <sstream>
#define N_TESTS 100
int main(int argc, char **argv)
{
/* read from the file */
if (argc < 2)
{
std::cerr << "Missing Hash File";
std::cerr << std::endl;
}
FILE *fd = fopen(argv[1], "r");
if (!fd)
{
std::cerr << "Failed to Open File: " << argv[1];
std::cerr << std::endl;
return 1;
}
char line[1000];
bdBloom filter;
int nHashes = 0;
while(fgets(line, 1000-1, fd))
{
std::string hash = line;
std::cerr << "Read Hash: " << hash;
std::cerr << std::endl;
filter.add(hash);
nHashes++;
}
fclose(fd);
filter.printFilter(std::cerr);
int bits = filter.countBits();
int filterBits = filter.filterBits();
std::cerr << "Filter Bits Set: " << bits;
std::cerr << std::endl;
double possible = ((double) bits) * bits;
double max = ((double) filterBits) * filterBits;
std::cerr << "Therefore Possible Finds: " << possible << "/" << max << " = %" << 100 * possible / max;
std::cerr << std::endl;
std::cerr << "With Insertions: " << nHashes;
std::cerr << std::endl;
std::cerr << std::endl;
std::cerr << std::endl;
std::cerr << "Filter String: " << filter.getFilter() << std::endl;
std::cerr << std::endl;
std::cerr << std::endl;
return 1;
}

165
libbitdht/src/tests/bdbloom_test.cc Normal file
View File

@ -0,0 +1,165 @@
/*
* bitdht/bdbloom_test.cc
*
* BitDHT: An Flexible DHT library.
*
* Copyright 2010 by Robert Fernie
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 3 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "bitdht@lunamutt.com".
*
*/
#include "util/bdbloom.h"
#include "bitdht/bdstddht.h"
#include <iostream>
#include <sstream>
#define N_TESTS 100
int main(int argc, char **argv)
{
/* Do this multiple times */
int i;
bdBloom filter;
std::list<bdNodeId> testIds;
std::list<bdNodeId>::iterator it;
for(i = 0; i < N_TESTS; i++)
{
bdNodeId targetId;
bdStdRandomNodeId(&targetId);
testIds.push_back(targetId);
}
std::cerr << "Test bdBloom Filter...." << std::endl;
for(it = testIds.begin(); it != testIds.end(); it++)
{
bdNodeId targetId = *it;
std::cerr << "-------------------------------------------------" << std::endl;
std::cerr << "Inserting : ";
std::ostringstream str;
bdStdPrintNodeId(str, &targetId);
std::cerr << str.str();
std::cerr << std::endl;
filter.add(str.str());
filter.printFilter(std::cerr);
}
std::string fs1 = filter.getFilter();
/* now extract, and reinsert filter */
bdBloom filter2;
filter2.setFilterBits(fs1);
std::string fs2 = filter2.getFilter();
filter2.printFilter(std::cerr);
if (fs1 == fs2)
{
std::cerr << "SUCCESS: Filter Correctly Transferred!";
std::cerr << std::endl;
}
else
{
std::cerr << "FAILURE: Filter Not Transferred!";
std::cerr << std::endl;
}
for(it = testIds.begin(); it != testIds.end(); it++)
{
bdNodeId targetId = *it;
std::cerr << "-------------------------------------------------" << std::endl;
std::cerr << "Testing : ";
std::cerr << std::endl;
std::ostringstream str;
bdStdPrintNodeId(str, &targetId);
std::cerr << str.str() << std::endl;
if (filter2.test(str.str()))
{
std::cerr << "SUCCESS: Filter Found Entry";
std::cerr << std::endl;
}
else
{
std::cerr << "FAILURE: Filter Didn't Found Entry";
std::cerr << std::endl;
}
}
int bits = filter.countBits();
int filterBits = filter.filterBits();
std::cerr << "Filter Bits Set: " << bits;
std::cerr << std::endl;
double possible = ((double) bits) * bits;
double max = ((double) filterBits) * filterBits;
std::cerr << "Therefore Possible Finds: " << possible << "/" << max << " = %" << 100 * possible / max;
std::cerr << std::endl;
std::cerr << "With Insertions: " << N_TESTS;
std::cerr << std::endl;
#define FINAL_TESTS (1000000)
int found = 0;
int didnt = 0;
for(i = 0; i < FINAL_TESTS; i++)
{
if ((i != 0) && (i % 100000 == 0))
{
std::cerr << "Run " << i << " Checks" << std::endl;
}
bdNodeId targetId;
bdStdRandomNodeId(&targetId);
std::ostringstream str;
bdStdPrintNodeId(str, &targetId);
if (filter2.test(str.str()))
{
found++;
}
else
{
didnt++;
}
}
std::cerr << "Final Stats: " << FINAL_TESTS << " random checks done";
std::cerr << std::endl;
std::cerr << "\t" << found << " " << 100.0 * ((double) found) / FINAL_TESTS << "% found";
std::cerr << std::endl;
std::cerr << "\t" << didnt << " " << 100.0 * ((double) didnt) / FINAL_TESTS << "% didnt";
std::cerr << std::endl;
return 1;
}

2
libbitdht/src/tests/bdnode_multitest1.cc
View File

@ -88,7 +88,7 @@ int main(int argc, char **argv)
for(j = 0; j < 5; j++)
{
int peeridx = bdRand::random_u32() % n_nodes;
int peeridx = bdRandom::random_u32() % n_nodes;
bdId pid = portIdx[peeridx];
node->addPotentialPeer(&pid, NULL);

3
libbitdht/src/tests/bdnode_test.cc
View File

@ -56,7 +56,7 @@ int main(int argc, char **argv)
node.printState();
#if 0
for(i = 0; i < N_QUERIES; i++)
{
/* create a query */
@ -65,6 +65,7 @@ int main(int argc, char **argv)
node.addQuery(&queryId, 0);
}
#endif
node.printState();

383
libbitdht/src/util/bdbloom.cc Normal file
View File

@ -0,0 +1,383 @@
/*
* bitdht/bdbloom.cc
*
* BitDHT: An Flexible DHT library.
*
* Copyright 2011 by Robert Fernie
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 3 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "bitdht@lunamutt.com".
*
*/
#include "util/bdbloom.h"
#include <sstream>
#include <iomanip>
/* Bloom Filter implementation */
bloomFilter::bloomFilter(int m, int k)
{
mBits.resize(m);
mHashFns.resize(k);
mFilterBits = m;
mNoHashs = k;
mNoElements = 0;
int i;
for(i = 0; i < m; i++)
{
mBits[i] = 0;
}
for(i = 0; i < k; i++)
{
mHashFns[i] = NULL;
}
}
uint8_t convertCharToUint8(char ch1, char ch2)
{
uint8_t value1 = 0;
uint8_t value2 = 0;
/* do char1 */
if (ch1 >= '0' && ch1 <= '9')
value1 = (ch1 - '0');
else if (ch1 >= 'A' && ch1 <= 'F')
value1 = (ch1 - 'A' + 10);
else if (ch1 >= 'a' && ch1 <= 'f')
value1 = (ch1 - 'a' + 10);
/* do char2 */
if (ch2 >= '0' && ch2 <= '9')
value2 = (ch2 - '0');
else if (ch2 >= 'A' && ch2 <= 'F')
value2 = (ch2 - 'A' + 10);
else if (ch2 >= 'a' && ch2 <= 'f')
value2 = (ch2 - 'a' + 10);
uint8_t output = (value1 << 4) + value2;
return output;
}
#define BITS_PER_BYTE (8)
int bloomFilter::setFilterBits(const std::string &hex)
{
int bytes = (mFilterBits / BITS_PER_BYTE);
if (mFilterBits % BITS_PER_BYTE)
{
bytes++;
}
if (hex.size() < bytes * 2)
{
return 0;
}
// convert to binary array.
uint8_t *tmparray = (uint8_t *) malloc(bytes);
int i = 0;
for(i = 0; i < bytes; i++)
{
tmparray[i] = convertCharToUint8(hex[2 * i], hex[2 * i + 1]);
}
for(i = 0; i < mFilterBits; i++)
{
int byte = i / BITS_PER_BYTE;
int bit = i % BITS_PER_BYTE;
uint8_t value = (tmparray[byte] & (1 << bit));
if (value)
{
mBits[i] = 1;
}
else
{
mBits[i] = 0;
}
}
free(tmparray);
return 1;
}
std::string bloomFilter::getFilter()
{
/* extract filter as a hex string */
std::string output;
int bytes = (mFilterBits / BITS_PER_BYTE);
if (mFilterBits % BITS_PER_BYTE)
{
bytes++;
}
// convert to binary array.
uint8_t *tmparray = (uint8_t *) malloc(bytes);
int i,j;
for(i = 0; i < bytes; i++)
{
tmparray[i] = 0;
for(j = 0; j < BITS_PER_BYTE; j++)
{
int bit = i * BITS_PER_BYTE + j;
if (mBits[bit])
{
tmparray[i] |= (1 << j);
}
}
}
std::ostringstream out;
for(int i = 0; i < bytes; i++)
{
out << std::setw(2) << std::setfill('0') << std::hex << (uint32_t) (tmparray)[i];
}
free(tmparray);
return out.str();
}
void bloomFilter::setBit(int bit)
{
mBits[bit] = 1;
}
bool bloomFilter::isBitSet(int bit)
{
return (mBits[bit] == 1);
}
uint32_t bloomFilter::filterBits()
{
return mFilterBits;
}
uint32_t bloomFilter::countBits()
{
int count = 0;
int i;
for(i = 0; i < mFilterBits; i++)
{
if (mBits[i])
{
count++;
}
}
return count;
}
void bloomFilter::printFilter(std::ostream &out)
{
out << "bloomFilter: m = " << mFilterBits;
out << " k = " << mNoHashs;
out << " n = " << mNoElements;
out << std::endl;
out << "BITS: ";
int i;
for(i = 0; i < mFilterBits; i++)
{
if ((i > 0) && (i % 32 == 0))
{
out << std::endl;
out << "BITS: ";
}
if (mBits[i])
{
out << "1";
}
else
{
out << "0";
}
}
out << std::endl;
out << "STR: " << getFilter();
out << std::endl;
}
void bloomFilter::setHashFunction(int idx, uint32_t (*hashfn)(const std::string &))
{
mHashFns[idx] = hashfn;
}
void bloomFilter::add(const std::string &hex)
{
uint32_t (*hashfn)(const std::string &);
int i;
for(i = 0; i < mNoHashs; i++)
{
hashfn = mHashFns[i];
int bit = hashfn(hex);
setBit(bit);
}
mNoElements++;
}
bool bloomFilter::test(const std::string &hex)
{
uint32_t (*hashfn)(const std::string &);
int i;
for(i = 0; i < mNoHashs; i++)
{
hashfn = mHashFns[i];
int bit = hashfn(hex);
if (!isBitSet(bit))
{
return false;
}
}
return true;
}
uint32_t getFirst10BitsAsNumber(const std::string &input)
{
if (input.size() < 8)
{
std::cerr << "getFirst10BitsAsNumber() ERROR Size too small!";
std::cerr << std::endl;
return 0;
}
uint8_t data[4];
data[0] = convertCharToUint8(input[0], input[1]);
data[1] = convertCharToUint8(input[2], input[3]);
data[2] = convertCharToUint8(input[4], input[5]);
data[3] = convertCharToUint8(input[6], input[7]);
uint32_t val = ((data[0] & 0xff) << 2) + ((data[1] & 0xc0) >> 6);
#ifdef DEBUG_BLOOM
std::cerr << "getFirst10BitsAsNumber() input: " << input;
std::cerr << std::endl;
std::cerr << "getFirst10BitsAsNumber() ";
std::cerr << " data[0]: " << std::hex << (uint32_t) data[0];
std::cerr << " data[1]: " << (uint32_t) data[1];
std::cerr << " data[2]: " << (uint32_t) data[2];
std::cerr << " data[3]: " << (uint32_t) data[3];
std::cerr << " val: " << std::dec << (uint32_t) val;
std::cerr << std::endl;
#endif
return val;
}
uint32_t getSecond10BitsAsNumber(const std::string &input)
{
if (input.size() < 8)
{
std::cerr << "getSecond10BitsAsNumber() ERROR Size too small!";
std::cerr << std::endl;
return 0;
}
uint8_t data[4];
data[0] = convertCharToUint8(input[0], input[1]);
data[1] = convertCharToUint8(input[2], input[3]);
data[2] = convertCharToUint8(input[4], input[5]);
data[3] = convertCharToUint8(input[6], input[7]);
uint32_t val = ((data[1] & 0x3f) << 4) + ((data[2] & 0xf0) >> 4);
#ifdef DEBUG_BLOOM
std::cerr << "getSecond10BitsAsNumber() input: " << input;
std::cerr << std::endl;
std::cerr << "getSecond10BitsAsNumber() ";
std::cerr << " data[0]: " << std::hex << (uint32_t) data[0];
std::cerr << " data[1]: " << (uint32_t) data[1];
std::cerr << " data[2]: " << (uint32_t) data[2];
std::cerr << " data[3]: " << (uint32_t) data[3];
std::cerr << " val: " << std::dec << (uint32_t) val;
std::cerr << std::endl;
#endif
return val;
}
uint32_t getMid10BitsAsNumber(const std::string &input)
{
if (input.size() < 8)
{
std::cerr << "getMid10BitsAsNumber() ERROR Size too small!";
std::cerr << std::endl;
return 0;
}
uint8_t data[4];
data[0] = convertCharToUint8(input[0], input[1]);
data[1] = convertCharToUint8(input[2], input[3]);
data[2] = convertCharToUint8(input[4], input[5]);
data[3] = convertCharToUint8(input[6], input[7]);
uint32_t val = ((data[0] & 0x07) << 7) + ((data[1] & 0x7f) >> 1);
#ifdef DEBUG_BLOOM
std::cerr << "getMid10BitsAsNumber() input: " << input;
std::cerr << std::endl;
std::cerr << "getMid10BitsAsNumber() ";
std::cerr << " data[0]: " << std::hex << (uint32_t) data[0];
std::cerr << " data[1]: " << (uint32_t) data[1];
std::cerr << " data[2]: " << (uint32_t) data[2];
std::cerr << " data[3]: " << (uint32_t) data[3];
std::cerr << " val: " << std::dec << (uint32_t) val;
std::cerr << std::endl;
#endif
return val;
}
#define BDFILTER_M 1024
#define BDFILTER_K 3
bdBloom::bdBloom()
:bloomFilter(BDFILTER_M, BDFILTER_K)
{
/* set the fns. */
setHashFunction(0, getFirst10BitsAsNumber);
setHashFunction(1, getSecond10BitsAsNumber);
setHashFunction(2, getMid10BitsAsNumber);
}

79
libbitdht/src/util/bdbloom.h Normal file
View File

@ -0,0 +1,79 @@
#ifndef BITDHT_BLOOM_H
#define BITDHT_BLOOM_H
/*
* bitdht/bdbloom.h
*
* BitDHT: An Flexible DHT library.
*
* Copyright 2011 by Robert Fernie
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 3 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "bitdht@lunamutt.com".
*
*/
#include <vector>
#include <string>
#include <iostream>
#include <inttypes.h>
class bloomFilter
{
public:
bloomFilter(int m, int k);
int setFilterBits(const std::string &hex);
std::string getFilter();
void printFilter(std::ostream &out);
bool test(const std::string &hex); // takes first m bits.
void add(const std::string &hex);
uint32_t countBits();
uint32_t filterBits();
protected:
void setHashFunction(int idx, uint32_t (*hashfn)(const std::string &));
private:
void setBit(int bit);
bool isBitSet(int bit);
std::vector<uint8_t> mBits;
std::vector<uint32_t (*)(const std::string &)> mHashFns;
uint32_t mFilterBits;
uint32_t mNoHashs;
uint32_t mNoElements;
};
/* our specific implementation */
class bdBloom: public bloomFilter
{
public:
bdBloom();
};
#endif