added serialisation/transmission of video data

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@7452 b45a01b8-16f6-495d-af2f-9b41ad6348cc

plugins/VOIP/gui/PluginGUIHandler.cpp

@@ -41,14 +41,25 @@ void PluginGUIHandler::ReceivedVoipData(const QString& qpeer_id)
 		if (cw) {
 			const QList<ChatWidgetHolder*> &chatWidgetHolderList = cw->chatWidgetHolderList();
 
-			foreach (ChatWidgetHolder *chatWidgetHolder, chatWidgetHolderList) {
+			foreach (ChatWidgetHolder *chatWidgetHolder, chatWidgetHolderList) 
+			{
 				VOIPChatWidgetHolder *acwh = dynamic_cast<VOIPChatWidgetHolder*>(chatWidgetHolder) ;
 
 				if (acwh) {
 					for (unsigned int i = 0; i < chunks.size(); ++i) {
-						for (unsigned int chunkIndex=0; chunkIndex<chunks.size(); chunkIndex++){
+						for (unsigned int chunkIndex=0; chunkIndex<chunks.size(); chunkIndex++)
+						{
 							QByteArray qb(reinterpret_cast<const char *>(chunks[chunkIndex].data),chunks[chunkIndex].size);
+
+							if(chunks[chunkIndex].type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO)
 								acwh->addAudioData(QString::fromStdString(peer_id.toStdString()),&qb);
+							else if(chunks[chunkIndex].type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO)
+							{
+								acwh->addVideoData(QString::fromStdString(peer_id.toStdString()),&qb);
+								std::cerr << "data triaged as video." << std::endl;
+							}
+							else
+								std::cerr << "Unknown data type received. type=" << chunks[chunkIndex].type << std::endl;
 						}
 					}
 					break;

plugins/VOIP/gui/QVideoDevice.cpp

@@ -74,10 +74,19 @@ void QVideoInputDevice::grabFrame()
 
 	QImage image = QImage((uchar*)img->imageData,img->width,img->height,QImage::Format_RGB888).scaled(QSize(_encoded_width,_encoded_height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ;
 
-	if(_video_encoder != NULL) _video_encoder->addImage(image) ;
+	if(_video_encoder != NULL) 
+	{
+		_video_encoder->addImage(image) ;
+		emit networkPacketReady() ;
+	}
 	if(_echo_output_device != NULL) _echo_output_device->showFrame(image) ;
 }
 
+bool QVideoInputDevice::getNextEncodedPacket(RsVoipDataChunk& chunk)
+{
+	return _video_encoder->nextPacket(chunk) ;
+}
+
 QVideoInputDevice::~QVideoInputDevice()
 {
 	stop() ;

plugins/VOIP/gui/QVideoDevice.h

@@ -1,6 +1,7 @@
 #pragma once 
 
 #include <QLabel>
+#include "interface/rsvoip.h"
 
 class VideoEncoder ;
 class CvCapture ;
@@ -36,17 +37,26 @@ class QVideoInputDevice: public QObject
 		//
 		void setEchoVideoTarget(QVideoOutputDevice *odev) { _echo_output_device = odev ; }
 	
+		// get the next encoded video data chunk.
+		//
+		bool getNextEncodedPacket(RsVoipDataChunk&) ;
+
 		void start() ;
 		void stop() ;
 
 	protected slots:
 		void grabFrame() ;
 
+	signals:
+		void networkPacketReady() ;
+
 	private:
 		VideoEncoder *_video_encoder ;
 		QTimer *_timer ;
 		CvCapture *_capture_device ;
 
 		QVideoOutputDevice *_echo_output_device ;
+
+		std::list<RsVoipDataChunk> _out_queue ;
 };
 

plugins/VOIP/gui/VOIPChatWidgetHolder.cpp

@@ -112,12 +112,15 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget)
 	videoWidget->layout()->addWidget(echoVideoDevice = new QVideoOutputDevice(videoWidget)) ;
 	videoWidget->layout()->addWidget(outputVideoDevice = new QVideoOutputDevice(videoWidget)) ;
 
+	connect(inputVideoDevice, SIGNAL(networkPacketReady()), this, SLOT(sendVideoData()));
+
 	echoVideoDevice->setMinimumSize(128,95) ;
 	outputVideoDevice->setMinimumSize(128,95) ;
 
 	mChatWidget->addChatHorizontalWidget(videoWidget) ;
 
 	inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ;
+	inputVideoDevice->setVideoEncoder(inputVideoProcessor) ;
 	outputVideoProcessor->setDisplayTarget(outputVideoDevice) ;
 }
 
@@ -221,6 +224,11 @@ void VOIPChatWidgetHolder::toggleVideoCapture()
 	}
 }
 
+void VOIPChatWidgetHolder::addVideoData(const QString name, QByteArray* array)
+{
+	outputVideoProcessor->receiveEncodedData((unsigned char *)array->data(),array->size()) ;
+}
+
 void VOIPChatWidgetHolder::addAudioData(const QString name, QByteArray* array)
 {
     if (!audioCaptureToggleButton->isChecked()) {
@@ -280,6 +288,17 @@ void VOIPChatWidgetHolder::addAudioData(const QString name, QByteArray* array)
     }
 }
 
+void VOIPChatWidgetHolder::sendVideoData()
+{
+	RsVoipDataChunk chunk ;
+
+	while(inputVideoDevice && inputVideoDevice->getNextEncodedPacket(chunk))
+	{
+		std::cerr << "Video data ready: sending it" << std::endl;
+		rsVoip->sendVoipData(mChatWidget->getPeerId(),chunk) ;
+	}
+}
+
 void VOIPChatWidgetHolder::sendAudioData()
 {
     while(inputAudioProcessor && inputAudioProcessor->hasPendingPackets()) {
@@ -287,6 +306,7 @@ void VOIPChatWidgetHolder::sendAudioData()
         RsVoipDataChunk chunk;
         chunk.size = qbarray.size();
         chunk.data = (void*)qbarray.constData();
+		  chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO ;
         rsVoip->sendVoipData(mChatWidget->getPeerId(),chunk);
     }
 }

plugins/VOIP/gui/VOIPChatWidgetHolder.h

@@ -34,6 +34,7 @@ private slots:
 
 public slots:
 	void sendAudioData();
+	void sendVideoData();
 
 protected:
 	// Audio input/output

plugins/VOIP/gui/VideoProcessor.cpp

@@ -24,8 +24,16 @@ bool VideoEncoder::addImage(const QImage& img)
 
 	encodeData(img) ;
 
-	if(_echo_output_device != NULL)
+	return true ;
-		_echo_output_device->showFrame(img) ;
+}
+
+bool VideoEncoder::nextPacket(RsVoipDataChunk& chunk)
+{
+	if(_out_queue.empty())
+		return false ;
+
+	chunk = _out_queue.front() ;
+	_out_queue.pop_front() ;
 
 	return true ;
 }
@@ -40,7 +48,10 @@ QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint
 	QByteArray qb((char*)encoded_image_data,size) ;
 	QImage image ;
 	if(image.loadFromData(qb))
+	{
+		std::cerr << "image decoded successfully" << std::endl;
 		return image ;
+	}
 	else
 		return QImage() ;
 }
@@ -55,6 +66,13 @@ void JPEGVideoEncoder::encodeData(const QImage& image)
 
 	//destination_decoder->receiveEncodedData((unsigned char *)qb.data(),qb.size()) ;
 
+	RsVoipDataChunk voip_chunk ;
+	voip_chunk.data = malloc(qb.size());
+	voip_chunk.size = qb.size() ;
+	voip_chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
+
+	_out_queue.push_back(voip_chunk) ;
+
 	std::cerr << "sending encoded data. size = " << qb.size() << std::endl;
 }
 

plugins/VOIP/gui/VideoProcessor.h

@@ -2,6 +2,7 @@
 
 #include <stdint.h>
 #include <QImage>
+#include "interface/rsvoip.h"
 
 class QVideoOutputDevice ;
 
@@ -43,20 +44,19 @@ class VideoDecoder
 class VideoEncoder
 {
 	public:
-		VideoEncoder() { _echo_output_device = NULL ;}
+		VideoEncoder() {}
 
 		// Takes the next image to be encoded. 
 		//
-		virtual bool addImage(const QImage& Image) ;
+		bool addImage(const QImage& Image) ;
 
+		bool packetReady() const { return !_out_queue.empty() ; }
+		bool nextPacket(RsVoipDataChunk& ) ;
 	protected:
 		//virtual bool sendEncodedData(unsigned char *mem,uint32_t size) = 0 ;
 		virtual void encodeData(const QImage& image) = 0 ;
 
-		unsigned char *buffer ;
+		std::list<RsVoipDataChunk> _out_queue ;
-		uint32_t buffer_size ;
-
-		QVideoOutputDevice *_echo_output_device ;
 };
 
 // Now derive various image encoding/decoding algorithms.

plugins/VOIP/interface/rsvoip.h

@@ -30,8 +30,11 @@ class RsVoipPongResult
 
 struct RsVoipDataChunk
 {
+	typedef enum { RS_VOIP_DATA_TYPE_AUDIO, RS_VOIP_DATA_TYPE_VIDEO } RsVoipDataType ;
+
 	void *data ; // create/delete using malloc/free.
 	uint32_t size ;
+	RsVoipDataType type ;	// video or audio
 };
 
 class RsVoip

plugins/VOIP/services/p3vors.cc

@@ -239,13 +239,24 @@ int p3VoRS::sendVoipData(const RsPeerId& peer_id,const RsVoipDataChunk& chunk)
 	if(item->voip_data == NULL)
 	{
 		std::cerr << "Cannot allocate RsVoipDataItem.voip_data of size " << chunk.size << " !" << std::endl;
+		delete item ;
 		return false ;
 	}
 	memcpy(item->voip_data,chunk.data,chunk.size) ;
-	item->flags = 0 ;
 	item->PeerId(peer_id) ;
 	item->data_size = chunk.size;
 
+	if(chunk.type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO) 
+		item->flags = RS_VOIP_FLAGS_AUDIO_DATA ;
+	else if(chunk.type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO) 
+		item->flags = RS_VOIP_FLAGS_VIDEO_DATA ;
+	else
+	{
+		std::cerr << "(EE) p3VoRs: cannot send chunk data. Unknown data type = " << chunk.type << std::endl;
+		delete item ;
+		return false ;
+	}
+
 	sendItem(item) ;
 
 	return true ;
@@ -339,6 +350,8 @@ void p3VoRS::handleData(RsVoipDataItem *item)
 
 	std::map<RsPeerId,VorsPeerInfo>::iterator it = mPeerInfo.find(item->PeerId()) ;
 
+	std::cerr << "Received VOIP data item. size = " << item->data_size << ", flags=" << item->flags <<std::endl;
+
 	if(it == mPeerInfo.end())
 	{
 		std::cerr << "Peer unknown to VOIP process. Dropping data" << std::endl;
@@ -370,6 +383,20 @@ bool p3VoRS::getIncomingData(const RsPeerId& peer_id,std::vector<RsVoipDataChunk
 		RsVoipDataChunk chunk ;
 		chunk.size = (*it2)->data_size ;
 		chunk.data = malloc((*it2)->data_size) ;
+
+		uint32_t type_flags = (*it2)->flags & (RS_VOIP_FLAGS_AUDIO_DATA | RS_VOIP_FLAGS_VIDEO_DATA) ;
+		if(type_flags == RS_VOIP_FLAGS_AUDIO_DATA)
+			chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO ;
+		else if(type_flags == RS_VOIP_FLAGS_VIDEO_DATA)
+			chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
+		else
+		{
+			std::cerr << "(EE) p3VoRS::getIncomingData(): error. Cannot handle item with unknown type " << type_flags << std::endl;
+			delete *it2 ;
+			free(chunk.data) ;
+			continue ;
+		}
+
 		memcpy(chunk.data,(*it2)->voip_data,(*it2)->data_size) ;
 
 		incoming_data_chunks.push_back(chunk) ;

plugins/VOIP/services/rsvoipitems.h

@@ -38,10 +38,14 @@ const uint16_t RS_SERVICE_TYPE_VOIP_PLUGIN = 0xa021;
 const uint8_t RS_PKT_SUBTYPE_VOIP_PING 	   = 0x01;
 const uint8_t RS_PKT_SUBTYPE_VOIP_PONG 	   = 0x02;
 const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL   = 0x03 ;
-const uint8_t RS_PKT_SUBTYPE_VOIP_DATA   	= 0x04 ;
+													     // 0x04 is unused because of a change in the protocol
+const uint8_t RS_PKT_SUBTYPE_VOIP_DATA      	= 0x05 ;
 
 const uint8_t QOS_PRIORITY_RS_VOIP = 9 ;
 
+const uint32_t RS_VOIP_FLAGS_VIDEO_DATA = 0x0001 ;
+const uint32_t RS_VOIP_FLAGS_AUDIO_DATA = 0x0002 ;
+
 class RsVoipItem: public RsItem
 {
 	public: