Merge remote branch 'origin/master'

.travis.yml

@@ -1,16 +1,13 @@
+sudo: required
+dist: trusty
 language: cpp
 
 compiler:
   - gcc
 
 before_install:
-  - echo $LANG
-  - echo $LC_ALL
   - sudo apt-get update
-  - sudo apt-get install build-essential checkinstall cmake g++ git libavutil-dev libavcodec-dev libavformat-dev libbz2-dev libcurl4-openssl-dev libdc1394-22-dev libglib2.0-dev libcv-dev libopencv-highgui-dev libhighgui-dev
-  - sudo apt-get install libgnome-keyring-dev libgstreamer-plugins-base0.10-dev libgstreamer0.10-dev libjasper-dev libjpeg-dev libmicrohttpd-dev libopencv-dev libprotobuf-dev libqt4-dev
-  - sudo apt-get install libspeex-dev libspeexdsp-dev libsqlite3-dev libssl-dev libswscale-dev
-  - sudo apt-get install libtbb-dev libtiff4-dev libupnp-dev libv4l-dev libxine-dev libxslt1-dev libxss-dev make pkg-config protobuf-compiler python-dev python-numpy subversion git yasm qtmobility-dev
+  - sudo apt-get install -y build-essential checkinstall cmake libavutil-dev libavcodec-dev libavformat-dev libbz2-dev libcurl4-openssl-dev libcv-dev libopencv-highgui-dev libhighgui-dev libgnome-keyring-dev libgstreamer-plugins-base0.10-dev libgstreamer0.10-dev libjasper-dev libjpeg-dev libmicrohttpd-dev libopencv-dev libprotobuf-dev libqt4-dev libspeex-dev libspeexdsp-dev libsqlite3-dev libssl-dev libswscale-dev libtbb-dev libtiff4-dev libupnp-dev libv4l-dev libxine-dev libxslt1-dev libxss-dev pkg-config protobuf-compiler python-dev qtmobility-dev
 #  - if [ $TRAVIS_OS_NAME == linux ]; then sudo apt-get update && sudo apt-get install -y llvm-3.4 llvm-3.4-dev; fi
 #  - rvm use $RVM --install --binary --fuzzy
 #  - gem update --system
@@ -32,7 +29,7 @@ addons:
     branch_pattern: coverity_scan
 
 before_script:
-  - qmake CONFIG+=NO_SQLCIPHER CONFIG+=tests
+  - qmake QMAKE_CC=$CC QMAKE_CXX=$CXX CONFIG+=NO_SQLCIPHER CONFIG+=tests
 
 #script: make
 script: if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make && tests/unittests/unittests >/dev/null 2>&1 ; fi
@@ -64,6 +61,3 @@ notifications:
 
 #env:
 #  - RVM=2.0.0 LANG="en_US.UTF-8"
-
-os:
-  - linux

libretroshare/src/pqi/pqissllistener.cc

@@ -322,11 +322,12 @@ int	pqissllistenbase::acceptconnection()
 	// can't be arsed making them all the time.
 	struct sockaddr_storage remote_addr;
 	socklen_t addrlen = sizeof(remote_addr);
-	int fd = accept(lsock, (struct sockaddr *) &remote_addr, &addrlen);
-	int err = 0;
 
 /********************************** WINDOWS/UNIX SPECIFIC PART ******************/
 #ifndef WINDOWS_SYS // ie UNIX
+	int fd = accept(lsock, (struct sockaddr *) &remote_addr, &addrlen);
+	int err = 0;
+    
         if (fd < 0)
         {
 		pqioutput(PQL_DEBUG_ALL, pqissllistenzone, 
@@ -347,7 +348,10 @@ int	pqissllistenbase::acceptconnection()
 
 /********************************** WINDOWS/UNIX SPECIFIC PART ******************/
 #else //WINDOWS_SYS 
-        if ((unsigned) fd == INVALID_SOCKET)
+	SOCKET fd = accept(lsock, (struct sockaddr *) &remote_addr, &addrlen);
+	int err = 0;
+    
+        if (fd == INVALID_SOCKET)
         {
 		pqioutput(PQL_DEBUG_ALL, pqissllistenzone, 
 		 "pqissllistenbase::acceptconnnection() Nothing to Accept!");

libretroshare/src/pqi/pqistreamer.cc

@@ -1024,6 +1024,7 @@ RsItem *pqistreamer::addPartialPacket(const void *block, uint32_t len, uint32_t
 	    {
 		    std::cerr << "(WW) dropping unfinished existing packet that gets to be replaced by new starting packet." << std::endl;
 		    free(rec.mem);
+            		rec.mem = NULL ;
 		    rec.size = 0 ;
 	    }
 	    // make sure this is a continuing packet, otherwise this is an error.

libretroshare/src/rsserver/p3face-info.cc

@@ -35,10 +35,54 @@
 #include <sqlcipher/sqlite3.h>
 #endif
 
+std::string RsServer::getSQLCipherVersion()
+{
+	sqlite3* mDb;
+	std::string versionstring("");
+	const char* version;
+	int rc = sqlite3_open_v2("", &mDb, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE , NULL); //create DB in a temp file
+
+	if(rc){
+		std::cerr << "Can't open database, Error code: " <<  sqlite3_errmsg(mDb)
+				  << std::endl;
+		sqlite3_close(mDb);
+		mDb = NULL;
+		return "";
+	 }
+
+	std::string sqlQuery = "PRAGMA cipher_version;";
+	sqlite3_stmt* stmt = NULL;
+	rc = sqlite3_prepare_v2(mDb, sqlQuery.c_str(), sqlQuery.length(), &stmt, NULL);
+	if (rc == SQLITE_OK) {
+		rc = sqlite3_step(stmt);
+		switch (rc) {
+		case SQLITE_ROW:
+			version = (const char *)sqlite3_column_text(stmt, 0); //not needed to free
+			versionstring.append(version);
+			break;
+		case SQLITE_DONE:
+			break;
+		default:
+			std::cerr << "RetroDb::tableExists(): Error executing statement (code: " << rc << ")"
+					  << std::endl;
+			break;
+		}
+	}
+
+	if (stmt) {
+		sqlite3_finalize(stmt);
+	}
+	sqlite3_close(mDb);	// no-op if mDb is NULL (https://www.sqlite.org/c3ref/close.html)
+	return versionstring;
+}
+
 void RsServer::getLibraries(std::list<RsLibraryInfo> &libraries)
 {
 	libraries.push_back(RsLibraryInfo("bzip2", BZ2_bzlibVersion()));
 	libraries.push_back(RsLibraryInfo("OpenSSL", SSLeay_version(SSLEAY_VERSION)));
-	libraries.push_back(RsLibraryInfo("SQLCipher", SQLITE_VERSION));
+	libraries.push_back(RsLibraryInfo("SQLite", SQLITE_VERSION));
+#ifndef NO_SQLCIPHER
+	libraries.push_back(RsLibraryInfo("SQLCipher", getSQLCipherVersion()));
+#endif
 	libraries.push_back(RsLibraryInfo("Zlib", ZLIB_VERSION));
 }

libretroshare/src/rsserver/p3face.h

@@ -139,6 +139,8 @@ class RsServer: public RsControl, public RsTickingThread
 
 	private: 
 
+		std::string getSQLCipherVersion();
+
 		// The real Server Parts.
 
 		//filedexserver *server;

libretroshare/src/turtle/p3turtle.cc

@@ -1192,7 +1192,7 @@ bool p3turtle::getTunnelServiceInfo(TurtleTunnelId tunnel_id,RsPeerId& vpid,RsFi
 
 		if(it == _incoming_file_hashes.end())
 		{
-			std::cerr << "p3turtle::handleRecvGenericTunnelItem(): hash " << hash << " for tunnel " << std::hex << it2->first << std::dec << " has no attached service! Dropping the item. This is a serious consistency error." << std::endl;
+			std::cerr << "p3turtle::handleRecvGenericTunnelItem(): hash " << hash << " for client side tunnel endpoint " << std::hex << tunnel_id << std::dec << " has been removed (probably a late response)! Dropping the item. " << std::endl;
 			return false;
 		}
 
@@ -1204,7 +1204,7 @@ bool p3turtle::getTunnelServiceInfo(TurtleTunnelId tunnel_id,RsPeerId& vpid,RsFi
 
 		if(it == _outgoing_file_hashes.end())
 		{
-			std::cerr << "p3turtle::handleRecvGenericTunnelItem(): hash " << hash << " for tunnel " << std::hex << it2->first  << std::dec<< " has no attached service! Dropping the item. This is a serious consistency error." << std::endl;
+			std::cerr << "p3turtle::handleRecvGenericTunnelItem(): hash " << hash << " for server side tunnel endpoint " << std::hex << tunnel_id << std::dec << " has been removed (probably a late response)! Dropping the item. " << std::endl;
 			return false;
 		}
 

libretroshare/src/util/retrodb.cc

@@ -67,7 +67,6 @@ RetroDb::RetroDb(const std::string &dbPath, int flags, const std::string& key) :
 			return;
 		 }
     }
-#endif
 
     char *err = NULL;
     rc = sqlite3_exec(mDb, "PRAGMA cipher_migrate;", NULL, NULL, &err);
@@ -81,8 +80,32 @@ RetroDb::RetroDb(const std::string &dbPath, int flags, const std::string& key) :
         std::cerr << std::endl;
         sqlite3_free(err);
     }
-}
 
+	//Test DB for correct sqlcipher version
+	if (sqlite3_exec(mDb, "PRAGMA user_version;", NULL, NULL, NULL) != SQLITE_OK)
+	{
+		std::cerr << "RetroDb::RetroDb(): Failed to open database: " << dbPath << std::endl << "Trying with settings for sqlcipher version 3...";
+		//Reopening the database with correct settings
+		rc = sqlite3_close(mDb);
+		mDb = NULL;
+		if(!rc)
+			rc = sqlite3_open_v2(dbPath.c_str(), &mDb, flags, NULL);
+		if(!rc && !mKey.empty())
+			rc = sqlite3_key(mDb, mKey.c_str(), mKey.size());
+		if(!rc)
+			rc = sqlite3_exec(mDb, "PRAGMA kdf_iter = 64000;", NULL, NULL, NULL);
+		if (!rc && (sqlite3_exec(mDb, "PRAGMA user_version;", NULL, NULL, NULL) == SQLITE_OK))
+		{
+			std::cerr << "\tSuccess" << std::endl;
+		} else {
+			std::cerr << "\tFailed, giving up" << std::endl;
+			sqlite3_close(mDb);
+			mDb = NULL;
+			return;
+		}
+	}
+#endif
+}
 
 RetroDb::~RetroDb(){
 

libretroshare/src/util/rsthreads.cc

@@ -40,6 +40,7 @@
  * #define DEBUG_THREADS 1
  * #define RSMUTEX_ABORT 1  // Catch wrong pthreads mode.
  *******/
+#define THREAD_DEBUG std::cerr << "[caller thread ID: " << std::hex << pthread_self() << ", thread ID: " << mTid << std::dec << "] "
 
 #ifdef RSMUTEX_ABORT
 	#include <stdlib.h>
@@ -60,14 +61,15 @@ void *RsThread::rsthread_init(void* p)
     // it is a replacement for pthread_join()
     pthread_detach(pthread_self());
 
+#ifdef DEBUG_THREADS
+    std::cerr << "[Thread ID:" << std::hex << pthread_self() << std::dec << "] thread is started. Calling runloop()..." << std::endl;
+#endif
+    
   thread -> runloop();
   return NULL;
 }
-RsThread::RsThread () : mMutex("RsThread")
+RsThread::RsThread() 
 {
-    sem_init(&mHasStoppedSemaphore,0,1) ;
-    sem_init(&mShouldStopSemaphore,0,0) ;
-
 #ifdef WINDOWS_SYS
     memset (&mTid, 0, sizeof(mTid));
 #else
@@ -77,33 +79,32 @@ RsThread::RsThread () : mMutex("RsThread")
 bool RsThread::isRunning()
 {
     // do we need a mutex for this ?
-    int sval =0;
-    sem_getvalue(&mHasStoppedSemaphore,&sval) ;
+    int sval = mHasStoppedSemaphore.value() ;
 
+#ifdef DEBUG_THREADS
+    THREAD_DEBUG << "  isRunning(): returning " << !sval << std::endl;
+#endif
     return !sval ;
 }
 
 bool RsThread::shouldStop()
 {
-        int sval =0;
-        sem_getvalue(&mShouldStopSemaphore,&sval) ;
-
+        int sval = mShouldStopSemaphore.value() ;
         return sval > 0;
 }
 
 void RsTickingThread::shutdown()
 {
 #ifdef DEBUG_THREADS
-    std::cerr << "pqithreadstreamer::stop()" << std::endl;
+    THREAD_DEBUG << "pqithreadstreamer::shutdown()" << std::endl;
 #endif
 
-    int sval =0;
-    sem_getvalue(&mHasStoppedSemaphore,&sval) ;
+    int sval = mHasStoppedSemaphore.value() ;
 
     if(sval > 0)
     {
 #ifdef DEBUG_THREADS
-        std::cerr << "  thread not running. Quit." << std::endl;
+        THREAD_DEBUG << "  thread not running. Quit." << std::endl;
 #endif
         return ;
     }
@@ -114,9 +115,9 @@ void RsTickingThread::shutdown()
 void RsThread::ask_for_stop()
 {
 #ifdef DEBUG_THREADS
-    std::cerr << "  calling stop" << std::endl;
+    THREAD_DEBUG << "  calling stop" << std::endl;
 #endif
-    sem_post(&mShouldStopSemaphore) ;
+    mShouldStopSemaphore.post();
 }
 
 void RsTickingThread::fullstop()
@@ -124,11 +125,11 @@ void RsTickingThread::fullstop()
     shutdown() ;
 
 #ifdef DEBUG_THREADS
-    std::cerr << "  waiting stop" << std::endl;
+    THREAD_DEBUG << "  waiting stop" << std::endl;
 #endif
-    sem_wait(&mHasStoppedSemaphore) ;
+    mHasStoppedSemaphore.wait();
 #ifdef DEBUG_THREADS
-    std::cerr << "  finished!" << std::endl;
+    THREAD_DEBUG << "  finished!" << std::endl;
 #endif
 }
 void RsThread::start()
@@ -136,14 +137,11 @@ void RsThread::start()
     pthread_t tid;
     void  *data = (void *)this ;
 
-    RS_STACK_MUTEX(mMutex) ;
-
 #ifdef DEBUG_THREADS
-    std::cerr << "pqithreadstreamer::run()" << std::endl;
-    std::cerr << "  initing should_stop=0" << std::endl;
-    std::cerr << "  initing has_stopped=1" << std::endl;
+    THREAD_DEBUG << "pqithreadstreamer::start() initing should_stop=0, has_stopped=1" << std::endl;
 #endif
-    sem_init(&mHasStoppedSemaphore,0,0) ;
+    mHasStoppedSemaphore.set(0) ;
+    mShouldStopSemaphore.set(0) ;
 
     int err ;
 
@@ -154,8 +152,8 @@ void RsThread::start()
         mTid = tid;
     else
     {
-        std::cerr << "Fatal error: pthread_create could not create a thread. Error returned: " << err << " !!!!!!!" << std::endl;
-    sem_init(&mHasStoppedSemaphore,0,1) ;
+        THREAD_DEBUG << "Fatal error: pthread_create could not create a thread. Error returned: " << err << " !!!!!!!" << std::endl;
+	mHasStoppedSemaphore.set(1) ;
     }
 }
 
@@ -163,33 +161,32 @@ void RsThread::start()
 
 RsTickingThread::RsTickingThread()
 {
-    sem_init(&mShouldStopSemaphore,0,0) ;
+#ifdef DEBUG_THREADS
+    THREAD_DEBUG << "RsTickingThread::RsTickingThread()" << std::endl;
+#endif
 }
 
 void RsSingleJobThread::runloop()
 {
-    sem_init(&mShouldStopSemaphore,0,0) ;
-
+    mShouldStopSemaphore.set(0) ;
     run() ;
 }
 
 void RsTickingThread::runloop()
 {
 #ifdef DEBUG_THREADS
-    std::cerr << "pqithreadstream::run()";
-    std::cerr << std::endl;
+    THREAD_DEBUG << "pqithreadstream::runloop()" << std::endl;
 #endif
-    sem_init(&mShouldStopSemaphore,0,0) ;
+    mShouldStopSemaphore.set(0) ;
 
     while(1)
     {
         if(shouldStop())
         {
 #ifdef DEBUG_THREADS
-            std::cerr << "pqithreadstreamer::run(): asked to stop." << std::endl;
-            std::cerr << "  setting hasStopped=1" << std::endl;
+            THREAD_DEBUG << "pqithreadstreamer::runloop(): asked to stop. setting hasStopped=1, and returning. Thread ends." << std::endl;
 #endif
-            sem_post(&mHasStoppedSemaphore) ;
+            mHasStoppedSemaphore.post();
             return ;
         }
 
@@ -217,9 +214,7 @@ void RsQueueThread::data_tick()
         mLastWork = now;
         mLastSleep = (uint32_t) (mMinSleep + (mLastSleep - mMinSleep) / 2.0);
 #ifdef DEBUG_THREADS
-        std::cerr << "RsQueueThread::run() done work: sleeping for: " << mLastSleep;
-        std::cerr << " ms";
-        std::cerr << std::endl;
+        THREAD_DEBUG << "RsQueueThread::data_tick() done work: sleeping for: " << mLastSleep << " ms" << std::endl;
 #endif
 
     }
@@ -235,9 +230,7 @@ void RsQueueThread::data_tick()
             mLastSleep = mMaxSleep;
         }
 #ifdef DEBUG_THREADS
-        std::cerr << "RsQueueThread::run() no work: sleeping for: " << mLastSleep;
-        std::cerr << " ms";
-        std::cerr << std::endl;
+        THREAD_DEBUG << "RsQueueThread::data_tick() no work: sleeping for: " << mLastSleep << " ms" << std::endl;
 #endif
     }
     usleep(mLastSleep * 1000); // mLastSleep msec

libretroshare/src/util/rsthreads.h

@@ -29,7 +29,9 @@
 #include <inttypes.h>
 #include <string>
 #include <iostream>
+#include <unistd.h>
 #include <semaphore.h>
+#include <util/rsmemory.h>
 
 /* RsIface Thread Wrappers */
 
@@ -168,6 +170,58 @@ class RsStackMutex
 //
 #define RS_STACK_MUTEX(m) RsStackMutex __local_retroshare_mutex(m,__PRETTY_FUNCTION__,__FILE__,__LINE__) 
 
+// This class handles a Mutex-based semaphore, that makes it cross plateform.
+class RsSemaphore
+{
+	class RsSemStruct
+	{
+	public:
+		RsSemStruct() : mtx("Semaphore mutex"), val(0) {}
+
+		RsMutex mtx ;
+		uint32_t val ;
+	};
+
+public:
+	RsSemaphore()
+	{
+		s = new RsSemStruct ;
+	}
+
+	~RsSemaphore()
+	{
+		delete s ;
+	}
+
+	void set(uint32_t i)
+	{
+		RS_STACK_MUTEX(s->mtx) ;
+		s->val = i ;
+	}
+
+	void post()
+	{
+		RS_STACK_MUTEX(s->mtx) ;
+		++(s->val) ;
+	}
+
+	uint32_t value()
+	{
+		RS_STACK_MUTEX(s->mtx) ;
+		return s->val ;
+	}
+
+	void wait()
+	{
+		while(value() == 0)
+			usleep(1000) ;
+
+		post() ;
+	}
+private:
+	RsSemStruct *s ;
+};
+
 class RsThread;
 
 /* to create a thread! */
@@ -198,11 +252,10 @@ class RsThread
 protected:
     virtual void runloop() =0; /* called once the thread is started. Should be overloaded by subclasses. */
 
-    sem_t mHasStoppedSemaphore;
-    sem_t mShouldStopSemaphore;
+    RsSemaphore mHasStoppedSemaphore;
+    RsSemaphore mShouldStopSemaphore;
 
     static void *rsthread_init(void*) ;
-    RsMutex   mMutex;
     pthread_t mTid;
 };
 

libretroshare/src/util/smallobject.cc

@@ -279,10 +279,6 @@ void *SmallObject::operator new(size_t size)
 #endif
 
 	RsStackMutex m(_mtx) ;
-
-	if(!_allocator._active)
-		return (void*)NULL;
-
 	void *p = _allocator.allocate(size) ;
 #ifdef DEBUG_MEMORY
 	std::cerr << "new RsItem: " << p << ", size=" << size << std::endl;