From 01c467c4c07563f265635ecddf4d6771cdd46dc6 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 15 Jul 2014 20:04:31 +0000 Subject: [PATCH] added serialisation/transmission of video data git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@7452 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- plugins/VOIP/gui/PluginGUIHandler.cpp | 25 ++++++++++----- plugins/VOIP/gui/QVideoDevice.cpp | 11 ++++++- plugins/VOIP/gui/QVideoDevice.h | 10 ++++++ plugins/VOIP/gui/VOIPChatWidgetHolder.cpp | 20 ++++++++++++ plugins/VOIP/gui/VOIPChatWidgetHolder.h | 1 + plugins/VOIP/gui/VideoProcessor.cpp | 24 +++++++++++++-- plugins/VOIP/gui/VideoProcessor.h | 12 ++++---- plugins/VOIP/interface/rsvoip.h | 15 +++++---- plugins/VOIP/services/p3vors.cc | 37 ++++++++++++++++++++--- plugins/VOIP/services/p3vors.h | 28 ++++++++--------- plugins/VOIP/services/rsvoipitems.h | 12 +++++--- 11 files changed, 149 insertions(+), 46 deletions(-) diff --git a/plugins/VOIP/gui/PluginGUIHandler.cpp b/plugins/VOIP/gui/PluginGUIHandler.cpp index 075b0ea24..a2209b635 100644 --- a/plugins/VOIP/gui/PluginGUIHandler.cpp +++ b/plugins/VOIP/gui/PluginGUIHandler.cpp @@ -24,31 +24,42 @@ void PluginGUIHandler::ReceivedVoipAccept(const QString& /*peer_id*/) void PluginGUIHandler::ReceivedVoipData(const QString& qpeer_id) { - std::cerr << "****** Plugin GUI handler: received VoipData!" << std::endl; + std::cerr << "****** Plugin GUI handler: received VoipData!" << std::endl; - RsPeerId peer_id(qpeer_id.toStdString()) ; + RsPeerId peer_id(qpeer_id.toStdString()) ; std::vector chunks ; - if(!rsVoip->getIncomingData(peer_id,chunks)) + if(!rsVoip->getIncomingData(peer_id,chunks)) { std::cerr << "PluginGUIHandler::ReceivedVoipData(): No data chunks to get. Weird!" << std::endl; return ; } - ChatDialog *di = ChatDialog::getExistingChat(peer_id) ; + ChatDialog *di = ChatDialog::getExistingChat(peer_id) ; if (di) { ChatWidget *cw = di->getChatWidget(); if (cw) { const QList &chatWidgetHolderList = cw->chatWidgetHolderList(); - foreach (ChatWidgetHolder *chatWidgetHolder, chatWidgetHolderList) { + foreach (ChatWidgetHolder *chatWidgetHolder, chatWidgetHolderList) + { VOIPChatWidgetHolder *acwh = dynamic_cast(chatWidgetHolder) ; if (acwh) { for (unsigned int i = 0; i < chunks.size(); ++i) { - for (unsigned int chunkIndex=0; chunkIndex(chunks[chunkIndex].data),chunks[chunkIndex].size); - acwh->addAudioData(QString::fromStdString(peer_id.toStdString()),&qb); + + if(chunks[chunkIndex].type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO) + acwh->addAudioData(QString::fromStdString(peer_id.toStdString()),&qb); + else if(chunks[chunkIndex].type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO) + { + acwh->addVideoData(QString::fromStdString(peer_id.toStdString()),&qb); + std::cerr << "data triaged as video." << std::endl; + } + else + std::cerr << "Unknown data type received. type=" << chunks[chunkIndex].type << std::endl; } } break; diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 386415020..dc47e91e0 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -74,10 +74,19 @@ void QVideoInputDevice::grabFrame() QImage image = QImage((uchar*)img->imageData,img->width,img->height,QImage::Format_RGB888).scaled(QSize(_encoded_width,_encoded_height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; - if(_video_encoder != NULL) _video_encoder->addImage(image) ; + if(_video_encoder != NULL) + { + _video_encoder->addImage(image) ; + emit networkPacketReady() ; + } if(_echo_output_device != NULL) _echo_output_device->showFrame(image) ; } +bool QVideoInputDevice::getNextEncodedPacket(RsVoipDataChunk& chunk) +{ + return _video_encoder->nextPacket(chunk) ; +} + QVideoInputDevice::~QVideoInputDevice() { stop() ; diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index d454aa635..d2e77b938 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -1,6 +1,7 @@ #pragma once #include +#include "interface/rsvoip.h" class VideoEncoder ; class CvCapture ; @@ -36,17 +37,26 @@ class QVideoInputDevice: public QObject // void setEchoVideoTarget(QVideoOutputDevice *odev) { _echo_output_device = odev ; } + // get the next encoded video data chunk. + // + bool getNextEncodedPacket(RsVoipDataChunk&) ; + void start() ; void stop() ; protected slots: void grabFrame() ; + signals: + void networkPacketReady() ; + private: VideoEncoder *_video_encoder ; QTimer *_timer ; CvCapture *_capture_device ; QVideoOutputDevice *_echo_output_device ; + + std::list _out_queue ; }; diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp index b914ef238..436eb161c 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp @@ -112,12 +112,15 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget) videoWidget->layout()->addWidget(echoVideoDevice = new QVideoOutputDevice(videoWidget)) ; videoWidget->layout()->addWidget(outputVideoDevice = new QVideoOutputDevice(videoWidget)) ; + connect(inputVideoDevice, SIGNAL(networkPacketReady()), this, SLOT(sendVideoData())); + echoVideoDevice->setMinimumSize(128,95) ; outputVideoDevice->setMinimumSize(128,95) ; mChatWidget->addChatHorizontalWidget(videoWidget) ; inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ; + inputVideoDevice->setVideoEncoder(inputVideoProcessor) ; outputVideoProcessor->setDisplayTarget(outputVideoDevice) ; } @@ -221,6 +224,11 @@ void VOIPChatWidgetHolder::toggleVideoCapture() } } +void VOIPChatWidgetHolder::addVideoData(const QString name, QByteArray* array) +{ + outputVideoProcessor->receiveEncodedData((unsigned char *)array->data(),array->size()) ; +} + void VOIPChatWidgetHolder::addAudioData(const QString name, QByteArray* array) { if (!audioCaptureToggleButton->isChecked()) { @@ -280,6 +288,17 @@ void VOIPChatWidgetHolder::addAudioData(const QString name, QByteArray* array) } } +void VOIPChatWidgetHolder::sendVideoData() +{ + RsVoipDataChunk chunk ; + + while(inputVideoDevice && inputVideoDevice->getNextEncodedPacket(chunk)) + { + std::cerr << "Video data ready: sending it" << std::endl; + rsVoip->sendVoipData(mChatWidget->getPeerId(),chunk) ; + } +} + void VOIPChatWidgetHolder::sendAudioData() { while(inputAudioProcessor && inputAudioProcessor->hasPendingPackets()) { @@ -287,6 +306,7 @@ void VOIPChatWidgetHolder::sendAudioData() RsVoipDataChunk chunk; chunk.size = qbarray.size(); chunk.data = (void*)qbarray.constData(); + chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO ; rsVoip->sendVoipData(mChatWidget->getPeerId(),chunk); } } diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.h b/plugins/VOIP/gui/VOIPChatWidgetHolder.h index 56637a3d7..8a528541e 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.h +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.h @@ -34,6 +34,7 @@ private slots: public slots: void sendAudioData(); + void sendVideoData(); protected: // Audio input/output diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 7e9b8e951..b7e715c7e 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -24,8 +24,16 @@ bool VideoEncoder::addImage(const QImage& img) encodeData(img) ; - if(_echo_output_device != NULL) - _echo_output_device->showFrame(img) ; + return true ; +} + +bool VideoEncoder::nextPacket(RsVoipDataChunk& chunk) +{ + if(_out_queue.empty()) + return false ; + + chunk = _out_queue.front() ; + _out_queue.pop_front() ; return true ; } @@ -40,7 +48,10 @@ QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint QByteArray qb((char*)encoded_image_data,size) ; QImage image ; if(image.loadFromData(qb)) + { + std::cerr << "image decoded successfully" << std::endl; return image ; + } else return QImage() ; } @@ -55,6 +66,13 @@ void JPEGVideoEncoder::encodeData(const QImage& image) //destination_decoder->receiveEncodedData((unsigned char *)qb.data(),qb.size()) ; - std::cerr <<"sending encoded data. size = " << qb.size() << std::endl; + RsVoipDataChunk voip_chunk ; + voip_chunk.data = malloc(qb.size()); + voip_chunk.size = qb.size() ; + voip_chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + _out_queue.push_back(voip_chunk) ; + + std::cerr << "sending encoded data. size = " << qb.size() << std::endl; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 6254f3777..e0bea8e83 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -2,6 +2,7 @@ #include #include +#include "interface/rsvoip.h" class QVideoOutputDevice ; @@ -43,20 +44,19 @@ class VideoDecoder class VideoEncoder { public: - VideoEncoder() { _echo_output_device = NULL ;} + VideoEncoder() {} // Takes the next image to be encoded. // - virtual bool addImage(const QImage& Image) ; + bool addImage(const QImage& Image) ; + bool packetReady() const { return !_out_queue.empty() ; } + bool nextPacket(RsVoipDataChunk& ) ; protected: //virtual bool sendEncodedData(unsigned char *mem,uint32_t size) = 0 ; virtual void encodeData(const QImage& image) = 0 ; - unsigned char *buffer ; - uint32_t buffer_size ; - - QVideoOutputDevice *_echo_output_device ; + std::list _out_queue ; }; // Now derive various image encoding/decoding algorithms. diff --git a/plugins/VOIP/interface/rsvoip.h b/plugins/VOIP/interface/rsvoip.h index 03b5605e8..c93e85dd5 100644 --- a/plugins/VOIP/interface/rsvoip.h +++ b/plugins/VOIP/interface/rsvoip.h @@ -30,24 +30,27 @@ class RsVoipPongResult struct RsVoipDataChunk { + typedef enum { RS_VOIP_DATA_TYPE_AUDIO, RS_VOIP_DATA_TYPE_VIDEO } RsVoipDataType ; + void *data ; // create/delete using malloc/free. uint32_t size ; + RsVoipDataType type ; // video or audio }; class RsVoip { public: - virtual int sendVoipHangUpCall(const RsPeerId& peer_id) = 0; - virtual int sendVoipRinging(const RsPeerId& peer_id) = 0; - virtual int sendVoipAcceptCall(const RsPeerId& peer_id) = 0; + virtual int sendVoipHangUpCall(const RsPeerId& peer_id) = 0; + virtual int sendVoipRinging(const RsPeerId& peer_id) = 0; + virtual int sendVoipAcceptCall(const RsPeerId& peer_id) = 0; // Sending data. The client keeps the memory ownership and must delete it after calling this. - virtual int sendVoipData(const RsPeerId& peer_id,const RsVoipDataChunk& chunk) = 0; + virtual int sendVoipData(const RsPeerId& peer_id,const RsVoipDataChunk& chunk) = 0; // The server fill in the data and gives up memory ownership. The client must delete the memory // in each chunk once it has been used. // - virtual bool getIncomingData(const RsPeerId& peer_id,std::vector& chunks) = 0; + virtual bool getIncomingData(const RsPeerId& peer_id,std::vector& chunks) = 0; typedef enum { AudioTransmitContinous = 0, AudioTransmitVAD = 1, AudioTransmitPushToTalk = 2 } enumAudioTransmit ; @@ -68,7 +71,7 @@ class RsVoip virtual bool getVoipEchoCancel() const = 0 ; virtual void setVoipEchoCancel(bool) = 0 ; - virtual uint32_t getPongResults(const RsPeerId& id, int n, std::list &results) = 0; + virtual uint32_t getPongResults(const RsPeerId& id, int n, std::list &results) = 0; }; diff --git a/plugins/VOIP/services/p3vors.cc b/plugins/VOIP/services/p3vors.cc index 29af687db..7652c926b 100644 --- a/plugins/VOIP/services/p3vors.cc +++ b/plugins/VOIP/services/p3vors.cc @@ -239,12 +239,23 @@ int p3VoRS::sendVoipData(const RsPeerId& peer_id,const RsVoipDataChunk& chunk) if(item->voip_data == NULL) { std::cerr << "Cannot allocate RsVoipDataItem.voip_data of size " << chunk.size << " !" << std::endl; + delete item ; return false ; } memcpy(item->voip_data,chunk.data,chunk.size) ; - item->flags = 0 ; - item->PeerId(peer_id) ; - item->data_size = chunk.size; + item->PeerId(peer_id) ; + item->data_size = chunk.size; + + if(chunk.type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO) + item->flags = RS_VOIP_FLAGS_AUDIO_DATA ; + else if(chunk.type == RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO) + item->flags = RS_VOIP_FLAGS_VIDEO_DATA ; + else + { + std::cerr << "(EE) p3VoRs: cannot send chunk data. Unknown data type = " << chunk.type << std::endl; + delete item ; + return false ; + } sendItem(item) ; @@ -337,7 +348,9 @@ void p3VoRS::handleData(RsVoipDataItem *item) // store the data in a queue. - std::map::iterator it = mPeerInfo.find(item->PeerId()) ; + std::map::iterator it = mPeerInfo.find(item->PeerId()) ; + + std::cerr << "Received VOIP data item. size = " << item->data_size << ", flags=" << item->flags <::iterator it = mPeerInfo.find(peer_id) ; + std::map::iterator it = mPeerInfo.find(peer_id) ; if(it == mPeerInfo.end()) { @@ -370,6 +383,20 @@ bool p3VoRS::getIncomingData(const RsPeerId& peer_id,std::vectordata_size ; chunk.data = malloc((*it2)->data_size) ; + + uint32_t type_flags = (*it2)->flags & (RS_VOIP_FLAGS_AUDIO_DATA | RS_VOIP_FLAGS_VIDEO_DATA) ; + if(type_flags == RS_VOIP_FLAGS_AUDIO_DATA) + chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_AUDIO ; + else if(type_flags == RS_VOIP_FLAGS_VIDEO_DATA) + chunk.type = RsVoipDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + else + { + std::cerr << "(EE) p3VoRS::getIncomingData(): error. Cannot handle item with unknown type " << type_flags << std::endl; + delete *it2 ; + free(chunk.data) ; + continue ; + } + memcpy(chunk.data,(*it2)->voip_data,(*it2)->data_size) ; incoming_data_chunks.push_back(chunk) ; diff --git a/plugins/VOIP/services/p3vors.h b/plugins/VOIP/services/p3vors.h index 1142dc785..757daf4e2 100644 --- a/plugins/VOIP/services/p3vors.h +++ b/plugins/VOIP/services/p3vors.h @@ -74,22 +74,22 @@ class p3VoRS: public RsPQIService, public RsVoip /***** overloaded from rsVoip *****/ - virtual uint32_t getPongResults(const RsPeerId &id, int n, std::list &results); + virtual uint32_t getPongResults(const RsPeerId &id, int n, std::list &results); // Call stuff. // // Sending data. The client keeps the memory ownership and must delete it after calling this. - virtual int sendVoipData(const RsPeerId &peer_id,const RsVoipDataChunk& chunk) ; + virtual int sendVoipData(const RsPeerId &peer_id,const RsVoipDataChunk& chunk) ; // The server fill in the data and gives up memory ownership. The client must delete the memory // in each chunk once it has been used. // - virtual bool getIncomingData(const RsPeerId& peer_id,std::vector& chunks) ; + virtual bool getIncomingData(const RsPeerId& peer_id,std::vector& chunks) ; - virtual int sendVoipHangUpCall(const RsPeerId& peer_id) ; - virtual int sendVoipRinging(const RsPeerId& peer_id) ; - virtual int sendVoipAcceptCall(const RsPeerId &peer_id) ; + virtual int sendVoipHangUpCall(const RsPeerId& peer_id) ; + virtual int sendVoipRinging(const RsPeerId& peer_id) ; + virtual int sendVoipAcceptCall(const RsPeerId &peer_id) ; /***** overloaded from p3Service *****/ /*! @@ -128,10 +128,10 @@ class p3VoRS: public RsPQIService, public RsVoip * chat msg items and custom status are saved */ virtual bool saveList(bool& cleanup, std::list&) ; - virtual bool loadList(std::list& load) ; - virtual std::string configurationFileName() const { return "voip.cfg" ; } + virtual bool loadList(std::list& load) ; + virtual std::string configurationFileName() const { return "voip.cfg" ; } - virtual RsServiceInfo getServiceInfo() ; + virtual RsServiceInfo getServiceInfo() ; private: int sendPackets(); @@ -141,24 +141,24 @@ class p3VoRS: public RsPQIService, public RsVoip int handlePing(RsVoipPingItem *item); int handlePong(RsVoipPongItem *item); - int storePingAttempt(const RsPeerId &id, double ts, uint32_t mCounter); - int storePongResult(const RsPeerId& id, uint32_t counter, double ts, double rtt, double offset); + int storePingAttempt(const RsPeerId &id, double ts, uint32_t mCounter); + int storePongResult(const RsPeerId& id, uint32_t counter, double ts, double rtt, double offset); void handleProtocol(RsVoipProtocolItem*) ; void handleData(RsVoipDataItem*) ; RsMutex mVorsMtx; - VorsPeerInfo *locked_GetPeerInfo(const RsPeerId& id); + VorsPeerInfo *locked_GetPeerInfo(const RsPeerId& id); static RsTlvKeyValue push_int_value(const std::string& key,int value) ; static int pop_int_value(const std::string& s) ; - std::map mPeerInfo; + std::map mPeerInfo; time_t mSentPingTime; uint32_t mCounter; - RsServiceControl *mServiceControl; + RsServiceControl *mServiceControl; PluginNotifier *mNotify ; int _atransmit ; diff --git a/plugins/VOIP/services/rsvoipitems.h b/plugins/VOIP/services/rsvoipitems.h index 19d79276a..452ffb635 100644 --- a/plugins/VOIP/services/rsvoipitems.h +++ b/plugins/VOIP/services/rsvoipitems.h @@ -35,13 +35,17 @@ const uint16_t RS_SERVICE_TYPE_VOIP_PLUGIN = 0xa021; -const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; -const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; -const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL= 0x03 ; -const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x04 ; +const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; +const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; +const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ; + // 0x04 is unused because of a change in the protocol +const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x05 ; const uint8_t QOS_PRIORITY_RS_VOIP = 9 ; +const uint32_t RS_VOIP_FLAGS_VIDEO_DATA = 0x0001 ; +const uint32_t RS_VOIP_FLAGS_AUDIO_DATA = 0x0002 ; + class RsVoipItem: public RsItem { public: