Merging from branches/v0.5-peernet/libbitdht (--- Merging r4354 through r4356 into '.')

* adding Bloom filter to speed up finding RS peers.
 * fixed startup bug. (loops into FAILURE)




git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@4357 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libbitdht/src/bitdht/bdaccount.cc

@@ -64,7 +64,7 @@ bdAccount::bdAccount()
 
 void bdAccount::incCounter(uint32_t idx, bool out)
 {
-	if ((idx < 0) || (idx > mNoStats-1))
+	if (idx > mNoStats-1)
 	{
 		std::cerr << "bdAccount::incCounter() Invalid Index";
 		std::cerr << std::endl;

libbitdht/src/bitdht/bdmanager.cc

@@ -74,6 +74,10 @@ bdNodeManager::bdNodeManager(bdNodeId *id, std::string dhtVersion, std::string b
         mNetworkSize = 0;
         mBdNetworkSize = 0;
 
+	std::string bfilter = "15108044002028000000200040400081a20840001000010400070240220000020501196b2420105218204008000000201102802041000a000004000004002881001180068301458000180000040000010080820e0005811000220200040800210280582001118024041002200004000000c44400080485a50008011084040200";
+
+	mBloomFilter.setFilterBits(bfilter);
+
 #ifdef DEBUG_MGR
 	std::cerr << "bdNodeManager::bdNodeManager() ID: ";
 	mFns->bdPrintNodeId(std::cerr, id);
@@ -379,11 +383,19 @@ void bdNodeManager::iteration()
 
 				status(); /* calculates mNetworkSize */
 
+#ifdef DEBUG_MGR
 				mAccount.printStats(std::cerr);
+#endif
 
 				/* Finally, Fail, and restart if we lose all peers */
-				if (mNetworkSize < MIN_OP_SPACE_SIZE)
+				uint32_t nodeSpaceSize = mNodeSpace.calcSpaceSize();
+				if (nodeSpaceSize < MIN_OP_SPACE_SIZE)
 				{
+					std::cerr << "bdNodeManager::iteration(): SpaceSize to Small: " << nodeSpaceSize;
+					std::cerr << std::endl;
+					std::cerr << "bdNodeManager::iteration(): REFRESH ==> FAILED";
+					std::cerr << std::endl;
+
 					mMode = BITDHT_MGR_STATE_FAILED;
 					mModeTS = now;
 				}
@@ -459,28 +471,30 @@ int bdNodeManager::QueryRandomLocalNet()
 		mQueryMgr->addWorthyPeerSource(&id); /* Tell BitDHT that we really want to ping their peers */
 		send_query(&id, &targetNodeId);
 			
+#ifdef DEBUG_NODE_MSGS
 		std::cerr << "bdNodeManager::QueryRandomLocalNet() Querying : ";
 		mFns->bdPrintId(std::cerr, &id);
 		std::cerr << " searching for : ";
 		mFns->bdPrintNodeId(std::cerr, &targetNodeId);
 		std::cerr << std::endl;
+#endif
 
 		if (isRandom)
 		{
+#ifdef DEBUG_NODE_MSGS
 			std::cerr << "bdNodeManager::QueryRandomLocalNet() Search is Random!";
 			std::cerr << std::endl;
+#endif
 		}
 
-#ifdef DEBUG_NODE_MSGS
-#endif
 		return 1;
 	}
 	else
 	{
 #ifdef DEBUG_NODE_MSGS
-#endif
 		std::cerr << "bdNodeManager::QueryRandomLocalNet() No LocalNet Peer Found";
 		std::cerr << std::endl;
+#endif
 	}
 
 	return 0;
@@ -525,7 +539,38 @@ void bdNodeManager::SearchForLocalNet()
 	{
 		/* install a new query */
 		bdNodeId targetNodeId;
-		bdStdRandomNodeId(&targetNodeId);
+
+		/* get something that filter approves of */
+		bool filterOk = false;
+		int i;
+
+#define MAX_FILTER_ATTEMPTS	3000
+
+		for(i = 0; (!filterOk) && (i < MAX_FILTER_ATTEMPTS); i++)
+		{
+			bdStdRandomNodeId(&targetNodeId);
+			std::ostringstream tststr;
+			bdStdPrintNodeId(tststr, &targetNodeId);
+
+			if (mBloomFilter.test(tststr.str()))
+			{
+				filterOk = true;
+			}
+		}
+
+		if (filterOk)
+		{
+			std::cerr << "bdNodeManager::SearchForLocalNet() " << i << " Attempts to find OkNode: ";
+			mFns->bdPrintNodeId(std::cerr, &targetNodeId);
+			std::cerr << std::endl;
+		}
+		else
+		{
+			std::cerr << "bdNodeManager::SearchForLocalNet() Failed to Find FilterOk this time: ";
+			mFns->bdPrintNodeId(std::cerr, &targetNodeId);
+			std::cerr << std::endl;
+		}
+
 
 		uint32_t qflags = BITDHT_QFLAGS_INTERNAL | BITDHT_QFLAGS_DISGUISE;
 		mQueryMgr->addQuery(&targetNodeId, qflags); 

libbitdht/src/bitdht/bdmanager.h

@@ -53,8 +53,9 @@
 #define BITDHT_PS_STATE_ONLINE		(0x00000400)
 #define BITDHT_PS_STATE_CONNECTED	(0x00000800)
 
-#include "bdiface.h"
+#include "bitdht/bdiface.h"
-#include "bdnode.h"
+#include "bitdht/bdnode.h"
+#include "util/bdbloom.h"
 
 
 
@@ -171,6 +172,8 @@ void    SearchForLocalNet();
 	uint32_t mNetworkSize;
 	uint32_t mBdNetworkSize;
 
+	bdBloom mBloomFilter;
+
 	/* future node functions */
 	//addPeerPing(foundId);
 	//clearPing(it->first);

libbitdht/src/bitdht/bdnode.cc

@@ -461,6 +461,18 @@ void bdNode::addPeer(const bdId *id, uint32_t peerflags)
 	// Finally we pass to connections for them to use.
 	mConnMgr->updatePotentialConnectionProxy(id, peerflags);
 
+
+//#define DISPLAY_BITDHTNODES	1
+#ifdef DISPLAY_BITDHTNODES
+	/* TEMP to extract IDS for BloomFilter */
+	if (peerflags & BITDHT_PEER_STATUS_DHT_ENGINE)
+	{
+		std::cerr << "bdNode::addPeer() FOUND BITDHT PEER";
+		std::cerr << std::endl;
+		mFns->bdPrintNodeId(std::cerr, &(id->id));
+		std::cerr << std::endl;
+	}
+#endif
 }
 
 /************************************ Process Remote Query *************************/

libbitdht/src/bitdht/bdpeer.cc

@@ -669,6 +669,7 @@ int	bdSpace::updateAttachedPeers()
 			}
 		}
 	}
+	return 1;
 }
 
 
@@ -1220,8 +1221,8 @@ bool bdSpace::findRandomPeerWithFlag(bdId &id, uint32_t withFlag)
 	uint32_t i = 0;
 	uint32_t buck = 0;
 
-	std::cerr << "bdSpace::findRandomPeerWithFlag()";
+	//std::cerr << "bdSpace::findRandomPeerWithFlag()";
-	std::cerr << std::endl;
+	//std::cerr << std::endl;
 
 	it = buckets.begin();
 	if (it != buckets.end())

libbitdht/src/bitdht/bdquerymgr.cc

@@ -297,6 +297,7 @@ int  bdQueryManager::getResults(bdNodeId *target, std::list<bdId> &answer, int q
 		/* will only be one matching query.. so end loop */
 		return results;
 	}
+	return 0;
 }
 
 		
@@ -328,10 +329,10 @@ void bdQueryManager::addWorthyPeerSource(bdId *src)
 	peer.mPeerId = *src;
 	peer.mFoundTime = now;
 
+#ifdef DEBUG_NODE_ACTIONS 
 	std::cerr << "bdQueryManager::addWorthyPeerSource(";
 	mFns->bdPrintId(std::cerr, src);
 	std::cerr << ")" << std::endl;
-#ifdef DEBUG_NODE_ACTIONS 
 #endif
 
 	mWorthyPeerSources.push_back(peer);
@@ -348,9 +349,11 @@ bool bdQueryManager::checkWorthyPeerSources(bdId *src)
 	{
 		if (now - it->mFoundTime > MAX_WORTHY_PEER_AGE)
 		{
+#ifdef DEBUG_NODE_ACTIONS 
 			std::cerr << "bdQueryManager::checkWorthyPeerSource() Discard old Source: ";
 			mFns->bdPrintId(std::cerr, &(it->mPeerId));
 			std::cerr << std::endl;
+#endif
 
 			it = mWorthyPeerSources.erase(it);
 		}
@@ -358,9 +361,11 @@ bool bdQueryManager::checkWorthyPeerSources(bdId *src)
 		{
 			if (it->mPeerId == *src)
 			{
-				//std::cerr << "bdQueryManager::checkWorthyPeerSource(";
+#ifdef DEBUG_NODE_ACTIONS 
-				//mFns->bdPrintId(std::cerr, src);
+				std::cerr << "bdQueryManager::checkWorthyPeerSource(";
-				//std::cerr << ") = true" << std::endl;
+				mFns->bdPrintId(std::cerr, src);
+				std::cerr << ") = true" << std::endl;
+#endif
 
 				return true;
 			}

libbitdht/src/bitdht/bdstore.cc

@@ -144,6 +144,7 @@ int     bdStore::filterIpList(const std::list<struct sockaddr_in> &filteredIPs)
 			}
 		}
 	}
+	return 1;
 }
 			
 		

libbitdht/src/libbitdht.pro

@@ -108,6 +108,7 @@ HEADERS += \
 	bitdht/bdfilter.h	\
 	bitdht/bdaccount.h	\
 	bitdht/bdquerymgr.h	\
+	util/bdbloom.h		\
 
 SOURCES += \
 	bitdht/bencode.c	\
@@ -131,5 +132,6 @@ SOURCES += \
 	bitdht/bdfilter.cc	\
 	bitdht/bdaccount.cc	\
 	bitdht/bdquerymgr.cc	\
+	util/bdbloom.cc		\
 
 

libbitdht/src/tests/Makefile

@@ -11,14 +11,19 @@ include $(TEST_TOP_DIR)/scripts/config.mk
 
 # Generic Test Harnesses.
 TESTOBJ  = bdmetric_test.o bdmsgs_test.o bdnode_test.o  bdspace_test.o 
-TESTOBJ  += bdmgr_multitest.o bdnode_multitest1.o bdquery_test.o bdstore_test.o
+TESTOBJ  += bdmgr_multitest.o bdquery_test.o bdstore_test.o
-TESTOBJ  += bdmidids_test.o bdnode_test2.o bdspace_test2.o udpbitdht_nettest.o
+TESTOBJ  += bdmidids_test.o bdspace_test2.o udpbitdht_nettest.o
+TESTOBJ  += bdbloom_test.o bdbloom_makefilter.o
 #TESTOBJ += bencode_test.o bdudp_test.o
+#TESTOBJ  = bdnode_test.o bdnode_multitest1.o bdnode_test2.o 
 
-TESTS  = bdmetric_test bdmsgs_test bdnode_test  bdspace_test 
+TESTS  = bdmetric_test bdmsgs_test bdspace_test 
-TESTS  += bdmgr_multitest bdnode_multitest1 bdquery_test bdstore_test
+TESTS  += bdmgr_multitest bdquery_test bdstore_test
-TESTS  += bdmidids_test bdnode_test2 bdspace_test2 udpbitdht_nettest
+TESTS  += bdmidids_test bdspace_test2 udpbitdht_nettest
+TESTS  += bdbloom_test 
 #TESTS  += bencode_test bdudp_test
+#Tests to Fix.
+#TESTS  = bdnode_test bdnode_multitest1 bdnode_test2 
 
 MANUAL_TESTS = 
 
@@ -68,6 +73,12 @@ udpbitdht_nettest: udpbitdht_nettest.o
 bencode_test: bencode_test.o
 	$(CC) $(CFLAGS) -o bencode_test bencode_test.o $(LIBS)
 
+bdbloom_test: bdbloom_test.o
+	$(CC) $(CFLAGS) -o bdbloom_test bdbloom_test.o $(LIBS)
+
+bdbloom_makefilter: bdbloom_makefilter.o
+	$(CC) $(CFLAGS) -o bdbloom_makefilter bdbloom_makefilter.o $(LIBS)
+
 
 clobber: remove_extra_files
 

libbitdht/src/tests/bdbloom_makefilter.cc

@@ -0,0 +1,91 @@
+/*
+ * bitdht/bdbloom_test.cc
+ *
+ * BitDHT: An Flexible DHT library.
+ *
+ * Copyright 2010 by Robert Fernie
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 3 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "bitdht@lunamutt.com".
+ *
+ */
+
+
+#include "util/bdbloom.h"
+#include "bitdht/bdstddht.h"
+#include <iostream>
+#include <sstream>
+
+#define N_TESTS 100
+
+int main(int argc, char **argv)
+{
+	/* read from the file */
+	if (argc < 2)
+	{
+		std::cerr << "Missing Hash File";
+		std::cerr << std::endl;
+	}
+
+	FILE *fd = fopen(argv[1], "r");
+	if (!fd)
+	{
+		std::cerr << "Failed to Open File: " << argv[1];
+		std::cerr << std::endl;
+		return 1;
+	}
+
+	char line[1000];
+
+	bdBloom filter;
+
+	int nHashes = 0;
+	while(fgets(line, 1000-1, fd))
+	{
+		std::string hash = line;
+		std::cerr << "Read Hash: " << hash;
+		std::cerr << std::endl;
+
+		filter.add(hash);
+		nHashes++;
+	}
+	fclose(fd);
+
+	filter.printFilter(std::cerr);
+
+	int bits = filter.countBits();
+	int filterBits = filter.filterBits();
+	std::cerr << "Filter Bits Set: " << bits;
+	std::cerr << std::endl;
+
+	double possible = ((double) bits) * bits;
+	double max = ((double) filterBits) * filterBits;
+	std::cerr << "Therefore Possible Finds: " << possible << "/" << max << " = %" << 100 * possible / max;
+	std::cerr << std::endl;
+	std::cerr << "With Insertions: " << nHashes;
+	std::cerr << std::endl;
+
+	std::cerr << std::endl;
+	std::cerr << std::endl;
+
+	std::cerr << "Filter String: " << filter.getFilter() << std::endl;
+	std::cerr << std::endl;
+	std::cerr << std::endl;
+
+	return 1;
+}
+
+

libbitdht/src/tests/bdbloom_test.cc

@@ -0,0 +1,165 @@
+/*
+ * bitdht/bdbloom_test.cc
+ *
+ * BitDHT: An Flexible DHT library.
+ *
+ * Copyright 2010 by Robert Fernie
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 3 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "bitdht@lunamutt.com".
+ *
+ */
+
+
+#include "util/bdbloom.h"
+#include "bitdht/bdstddht.h"
+#include <iostream>
+#include <sstream>
+
+#define N_TESTS 100
+
+int main(int argc, char **argv)
+{
+
+	/* Do this multiple times */
+	int i;
+
+
+	bdBloom filter;
+
+	std::list<bdNodeId> testIds;
+	std::list<bdNodeId>::iterator it;
+
+	for(i = 0; i < N_TESTS; i++)
+	{
+		bdNodeId targetId;
+		bdStdRandomNodeId(&targetId);
+		testIds.push_back(targetId);
+	}
+
+	std::cerr << "Test bdBloom Filter...." << std::endl;
+	for(it = testIds.begin(); it != testIds.end(); it++)
+	{
+		bdNodeId targetId = *it;
+	
+		std::cerr << "-------------------------------------------------" << std::endl;
+		std::cerr << "Inserting : ";
+		std::ostringstream str;
+		bdStdPrintNodeId(str, &targetId);
+		std::cerr << str.str();
+		std::cerr << std::endl;
+
+		filter.add(str.str());
+
+		filter.printFilter(std::cerr);
+	}
+
+	std::string fs1 = filter.getFilter();
+	/* now extract, and reinsert filter */
+	bdBloom filter2;
+	filter2.setFilterBits(fs1);
+
+	std::string fs2 = filter2.getFilter();
+	filter2.printFilter(std::cerr);
+
+	if (fs1 == fs2)
+	{
+		std::cerr << "SUCCESS: Filter Correctly Transferred!";
+		std::cerr << std::endl;
+	}
+	else
+	{
+		std::cerr << "FAILURE: Filter Not Transferred!";
+		std::cerr << std::endl;
+	}
+
+
+	for(it = testIds.begin(); it != testIds.end(); it++)
+	{
+		bdNodeId targetId = *it;
+	
+		std::cerr << "-------------------------------------------------" << std::endl;
+		std::cerr << "Testing : ";
+		std::cerr << std::endl;
+
+		std::ostringstream str;
+		bdStdPrintNodeId(str, &targetId);
+		std::cerr << str.str() << std::endl;
+
+		if (filter2.test(str.str()))
+		{
+			std::cerr << "SUCCESS: Filter Found Entry";
+			std::cerr << std::endl;
+		}
+		else
+		{
+			std::cerr << "FAILURE: Filter Didn't Found Entry";
+			std::cerr << std::endl;
+		}
+	}
+
+	int bits = filter.countBits();
+	int filterBits = filter.filterBits();
+	std::cerr << "Filter Bits Set: " << bits;
+	std::cerr << std::endl;
+
+	double possible = ((double) bits) * bits;
+	double max = ((double) filterBits) * filterBits;
+	std::cerr << "Therefore Possible Finds: " << possible << "/" << max << " = %" << 100 * possible / max;
+	std::cerr << std::endl;
+	std::cerr << "With Insertions: " << N_TESTS;
+	std::cerr << std::endl;
+
+#define FINAL_TESTS	(1000000)
+	int found = 0;
+	int didnt = 0;
+	for(i = 0; i < FINAL_TESTS; i++)
+	{
+		if ((i != 0) && (i % 100000 == 0))
+		{
+			std::cerr << "Run " << i << " Checks" << std::endl;
+		}
+
+		bdNodeId targetId;
+		bdStdRandomNodeId(&targetId);
+		std::ostringstream str;
+		bdStdPrintNodeId(str, &targetId);
+
+		if (filter2.test(str.str()))
+		{
+			found++;
+		}
+		else
+		{
+			didnt++;
+		}
+	}
+
+	std::cerr << "Final Stats: " << FINAL_TESTS << " random checks done";
+	std::cerr << std::endl;
+	std::cerr << "\t" << found << "   " << 100.0 * ((double) found) / FINAL_TESTS << "% found";
+	std::cerr << std::endl;
+	std::cerr << "\t" << didnt << "   " << 100.0 * ((double) didnt) / FINAL_TESTS << "% didnt";
+	std::cerr << std::endl;
+
+
+	
+		
+
+	return 1;
+}
+
+

libbitdht/src/tests/bdnode_multitest1.cc

@@ -88,7 +88,7 @@ int main(int argc, char **argv)
 
 		for(j = 0; j < 5; j++)
 		{		
-			int peeridx = bdRand::random_u32() % n_nodes;
+			int peeridx = bdRandom::random_u32() % n_nodes;
 
 			bdId pid = portIdx[peeridx];
 			node->addPotentialPeer(&pid, NULL);

libbitdht/src/tests/bdnode_test.cc

@@ -56,7 +56,7 @@ int main(int argc, char **argv)
 
 	node.printState();
 
-
+#if 0
 	for(i = 0; i < N_QUERIES; i++)
 	{
 		/* create a query */
@@ -65,6 +65,7 @@ int main(int argc, char **argv)
 
 		node.addQuery(&queryId, 0);
 	}
+#endif
 
 	node.printState();
 

libbitdht/src/util/bdbloom.cc

@@ -0,0 +1,383 @@
+/*
+ * bitdht/bdbloom.cc
+ *
+ * BitDHT: An Flexible DHT library.
+ *
+ * Copyright 2011 by Robert Fernie
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 3 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "bitdht@lunamutt.com".
+ *
+ */
+
+#include "util/bdbloom.h"
+
+#include <sstream>
+#include <iomanip>
+
+
+/* Bloom Filter implementation */
+
+
+bloomFilter::bloomFilter(int m, int k)
+{
+	mBits.resize(m);
+	mHashFns.resize(k);
+
+	mFilterBits = m;
+	mNoHashs = k;
+	mNoElements = 0;
+
+	int i;
+	for(i = 0; i < m; i++)
+	{
+		mBits[i] = 0;
+	}
+
+	for(i = 0; i < k; i++)
+	{
+		mHashFns[i] = NULL;
+	}
+
+}
+
+uint8_t convertCharToUint8(char ch1, char ch2)
+{
+	uint8_t value1 = 0;
+	uint8_t value2 = 0;
+	
+	/* do char1 */
+	if (ch1 >= '0' && ch1 <= '9')
+	value1 = (ch1 - '0');
+	else if (ch1 >= 'A' && ch1 <= 'F')
+	value1 = (ch1 - 'A' + 10);
+	else if (ch1 >= 'a' && ch1 <= 'f')
+	value1 = (ch1 - 'a' + 10);
+	
+	/* do char2 */
+	if (ch2 >= '0' && ch2 <= '9')
+	value2 = (ch2 - '0');
+	else if (ch2 >= 'A' && ch2 <= 'F')
+	value2 = (ch2 - 'A' + 10);
+	else if (ch2 >= 'a' && ch2 <= 'f')
+	value2 = (ch2 - 'a' + 10);
+	
+	uint8_t output = (value1 << 4) + value2;
+	return output;
+}
+
+#define BITS_PER_BYTE (8)
+
+int bloomFilter::setFilterBits(const std::string &hex)
+{
+	int bytes = (mFilterBits / BITS_PER_BYTE);
+	if (mFilterBits % BITS_PER_BYTE)
+	{
+		bytes++;
+	}
+
+	if (hex.size() < bytes * 2)
+	{
+		return 0;
+	}
+
+	// convert to binary array.
+	uint8_t *tmparray = (uint8_t *) malloc(bytes);
+	int i = 0;
+
+	for(i = 0; i < bytes; i++)
+	{
+		tmparray[i] = convertCharToUint8(hex[2 * i], hex[2 * i + 1]);
+	}
+
+
+	for(i = 0; i < mFilterBits; i++)
+	{
+		int byte = i / BITS_PER_BYTE;
+		int bit = i % BITS_PER_BYTE;
+		uint8_t value = (tmparray[byte] & (1 << bit));
+
+		if (value)
+		{
+			mBits[i] = 1;
+		}
+		else
+		{
+			mBits[i] = 0;
+		}
+	}
+
+	free(tmparray);
+	return 1;
+}
+
+std::string bloomFilter::getFilter()
+{
+	/* extract filter as a hex string */
+	std::string output;
+	int bytes = (mFilterBits / BITS_PER_BYTE);
+	if (mFilterBits % BITS_PER_BYTE)
+	{
+		bytes++;
+	}
+
+	// convert to binary array.
+	uint8_t *tmparray = (uint8_t *) malloc(bytes);
+	int i,j;
+	
+	for(i = 0; i < bytes; i++)
+	{
+		tmparray[i] = 0;
+		for(j = 0; j < BITS_PER_BYTE; j++)
+		{
+			int bit = i * BITS_PER_BYTE + j;
+			if (mBits[bit])
+			{
+				tmparray[i] |= (1 << j);
+			}
+		}
+	}
+
+	std::ostringstream out;
+	for(int i = 0; i < bytes; i++)
+	{
+		out << std::setw(2) << std::setfill('0') << std::hex << (uint32_t) (tmparray)[i];
+	}
+
+	free(tmparray);
+
+	return out.str();
+}
+
+void bloomFilter::setBit(int bit)
+{
+	mBits[bit] = 1;
+
+}
+
+
+bool bloomFilter::isBitSet(int bit)
+{
+	return (mBits[bit] == 1);
+}
+
+uint32_t bloomFilter::filterBits()
+{
+	return mFilterBits;
+}
+
+uint32_t bloomFilter::countBits()
+{
+	int count = 0;
+	int i;
+	for(i = 0; i < mFilterBits; i++)
+	{
+		if (mBits[i])
+		{
+			count++;
+		}
+	}
+	return count;
+}
+
+
+void bloomFilter::printFilter(std::ostream &out)
+{
+	out << "bloomFilter: m = " << mFilterBits;
+	out << " k = " << mNoHashs;
+	out << " n = " << mNoElements;
+	out << std::endl;
+
+	out << "BITS: ";
+	int i;
+	for(i = 0; i < mFilterBits; i++)
+	{
+		if ((i > 0) && (i % 32 == 0))
+		{
+			out << std::endl;
+			out << "BITS: ";
+		}
+		if (mBits[i])
+		{
+			out << "1";
+		}
+		else
+		{
+			out << "0";
+		}
+	}
+	out << std::endl;
+	out << "STR: " << getFilter();
+	out << std::endl;
+}
+
+void bloomFilter::setHashFunction(int idx,  uint32_t (*hashfn)(const std::string &))
+{
+	mHashFns[idx] = hashfn;
+}
+
+void bloomFilter::add(const std::string &hex)
+{
+	uint32_t (*hashfn)(const std::string &);
+	int i;
+	for(i = 0; i < mNoHashs; i++)
+	{
+		hashfn = mHashFns[i];
+
+		int bit = hashfn(hex);
+
+		setBit(bit);
+	}
+
+	mNoElements++;
+
+}
+
+bool bloomFilter::test(const std::string &hex)
+{
+	uint32_t (*hashfn)(const std::string &);
+	int i;
+	for(i = 0; i < mNoHashs; i++)
+	{
+		hashfn = mHashFns[i];
+
+		int bit = hashfn(hex);
+
+		if (!isBitSet(bit))
+		{
+			return false;
+		}
+	}
+	return true;
+}
+
+uint32_t getFirst10BitsAsNumber(const std::string &input)
+{
+	if (input.size() < 8)
+	{
+		std::cerr << "getFirst10BitsAsNumber() ERROR Size too small!";
+		std::cerr << std::endl;
+		return 0;
+	}
+
+	uint8_t data[4];
+
+	data[0] = convertCharToUint8(input[0], input[1]);
+	data[1] = convertCharToUint8(input[2], input[3]);
+	data[2] = convertCharToUint8(input[4], input[5]);
+	data[3] = convertCharToUint8(input[6], input[7]);
+
+	uint32_t val = ((data[0] & 0xff) << 2) + ((data[1] & 0xc0) >> 6);
+
+#ifdef DEBUG_BLOOM
+	std::cerr << "getFirst10BitsAsNumber() input: " << input;
+	std::cerr << std::endl;
+	std::cerr << "getFirst10BitsAsNumber() ";
+	std::cerr << " data[0]: " << std::hex << (uint32_t) data[0];
+	std::cerr << " data[1]: " << (uint32_t) data[1];
+	std::cerr << " data[2]: " << (uint32_t) data[2];
+	std::cerr << " data[3]: " << (uint32_t) data[3];
+	std::cerr << " val: " << std::dec << (uint32_t) val;
+	std::cerr << std::endl;
+#endif
+
+	return val;
+}
+
+uint32_t getSecond10BitsAsNumber(const std::string &input)
+{
+	if (input.size() < 8)
+	{
+		std::cerr << "getSecond10BitsAsNumber() ERROR Size too small!";
+		std::cerr << std::endl;
+		return 0;
+	}
+
+	uint8_t data[4];
+
+	data[0] = convertCharToUint8(input[0], input[1]);
+	data[1] = convertCharToUint8(input[2], input[3]);
+	data[2] = convertCharToUint8(input[4], input[5]);
+	data[3] = convertCharToUint8(input[6], input[7]);
+
+	uint32_t val = ((data[1] & 0x3f) << 4) + ((data[2] & 0xf0) >> 4);
+
+#ifdef DEBUG_BLOOM
+	std::cerr << "getSecond10BitsAsNumber() input: " << input;
+	std::cerr << std::endl;
+	std::cerr << "getSecond10BitsAsNumber() ";
+	std::cerr << " data[0]: " << std::hex << (uint32_t) data[0];
+	std::cerr << " data[1]: " << (uint32_t) data[1];
+	std::cerr << " data[2]: " << (uint32_t) data[2];
+	std::cerr << " data[3]: " << (uint32_t) data[3];
+	std::cerr << " val: " << std::dec << (uint32_t) val;
+	std::cerr << std::endl;
+#endif
+
+	return val;
+}
+
+
+uint32_t getMid10BitsAsNumber(const std::string &input)
+{
+	if (input.size() < 8)
+	{
+		std::cerr << "getMid10BitsAsNumber() ERROR Size too small!";
+		std::cerr << std::endl;
+		return 0;
+	}
+
+	uint8_t data[4];
+
+	data[0] = convertCharToUint8(input[0], input[1]);
+	data[1] = convertCharToUint8(input[2], input[3]);
+	data[2] = convertCharToUint8(input[4], input[5]);
+	data[3] = convertCharToUint8(input[6], input[7]);
+
+	uint32_t val = ((data[0] & 0x07) << 7) + ((data[1] & 0x7f) >> 1);
+
+#ifdef DEBUG_BLOOM
+	std::cerr << "getMid10BitsAsNumber() input: " << input;
+	std::cerr << std::endl;
+	std::cerr << "getMid10BitsAsNumber() ";
+	std::cerr << " data[0]: " << std::hex << (uint32_t) data[0];
+	std::cerr << " data[1]: " << (uint32_t) data[1];
+	std::cerr << " data[2]: " << (uint32_t) data[2];
+	std::cerr << " data[3]: " << (uint32_t) data[3];
+	std::cerr << " val: " << std::dec << (uint32_t) val;
+	std::cerr << std::endl;
+#endif
+
+	return val;
+}
+
+
+#define BDFILTER_M 1024
+#define BDFILTER_K 3
+
+bdBloom::bdBloom()
+	:bloomFilter(BDFILTER_M, BDFILTER_K)
+{
+	/* set the fns. */
+	setHashFunction(0, getFirst10BitsAsNumber);
+	setHashFunction(1, getSecond10BitsAsNumber);
+	setHashFunction(2, getMid10BitsAsNumber);
+}
+
+
+
+
+

libbitdht/src/util/bdbloom.h

@@ -0,0 +1,79 @@
+#ifndef BITDHT_BLOOM_H
+#define BITDHT_BLOOM_H
+
+/*
+ * bitdht/bdbloom.h
+ *
+ * BitDHT: An Flexible DHT library.
+ *
+ * Copyright 2011 by Robert Fernie
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License Version 3 as published by the Free Software Foundation.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA.
+ *
+ * Please report all bugs and problems to "bitdht@lunamutt.com".
+ *
+ */
+
+
+#include <vector>
+#include <string>
+
+#include <iostream>
+#include <inttypes.h>
+
+
+class bloomFilter
+{
+	public:
+
+	bloomFilter(int m, int k);
+
+int 	setFilterBits(const std::string &hex);
+std::string getFilter();
+
+void 	printFilter(std::ostream &out);
+
+bool 	test(const std::string &hex);  // takes first m bits.
+void 	add(const std::string &hex);
+uint32_t countBits();
+uint32_t filterBits();
+
+	protected:
+void 	setHashFunction(int idx,  uint32_t (*hashfn)(const std::string &));
+
+	private:
+void 	setBit(int bit);
+bool 	isBitSet(int bit);
+
+	std::vector<uint8_t> mBits;
+	std::vector<uint32_t (*)(const std::string &)> mHashFns;
+
+	uint32_t mFilterBits;
+	uint32_t mNoHashs;
+	uint32_t mNoElements;
+};
+
+
+/* our specific implementation */
+class bdBloom: public bloomFilter
+{
+	public:
+
+	bdBloom();
+};
+
+
+#endif
+