diff --git a/build_scripts/Debian+Ubuntu/control.precise b/build_scripts/Debian+Ubuntu/control.precise index b008cf5fa..70d77ea50 100644 --- a/build_scripts/Debian+Ubuntu/control.precise +++ b/build_scripts/Debian+Ubuntu/control.precise @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 b/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 index 244bdb2a0..c72507d0f 100644 --- a/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 +++ b/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/control.ubuntu_lucid b/build_scripts/Debian+Ubuntu/control.ubuntu_lucid index 1188ca8a0..ae12f4247 100644 --- a/build_scripts/Debian+Ubuntu/control.ubuntu_lucid +++ b/build_scripts/Debian+Ubuntu/control.ubuntu_lucid @@ -2,7 +2,7 @@ Source: retroshare Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev, libavcodec-dev Standards-Version: 3.9.1 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/debian/control b/build_scripts/Debian+Ubuntu/debian/control index 357150da5..b251bb923 100644 --- a/build_scripts/Debian+Ubuntu/debian/control +++ b/build_scripts/Debian+Ubuntu/debian/control @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net diff --git a/plugins/VOIP/VOIP.pro b/plugins/VOIP/VOIP.pro index e7f054553..86a716309 100644 --- a/plugins/VOIP/VOIP.pro +++ b/plugins/VOIP/VOIP.pro @@ -96,4 +96,4 @@ TRANSLATIONS += \ lang/VOIP_tr.ts \ lang/VOIP_zh_CN.ts -LIBS += -lspeex -lspeexdsp +LIBS += -lspeex -lspeexdsp -lavformat -lavcodec -lavutil diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index e23083200..d1503040b 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -29,17 +29,15 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -//#include "AudioInput.h" -//#include "AudioOutput.h" #include "AudioStats.h" #include "AudioInputConfig.h" -//#include "Global.h" -//#include "NetworkConfig.h" #include "audiodevicehelper.h" #include "AudioWizard.h" +#include "gui/VideoProcessor.h" #include "gui/common/RSGraphWidget.h" +#include "util/RsProtectedTimer.h" + #include #define iroundf(x) ( static_cast(x) ) @@ -55,9 +53,9 @@ void AudioInputDialog::showEvent(QShowEvent *) { class voipGraphSource: public RSGraphSource { public: - voipGraphSource() {} + voipGraphSource() : video_input(NULL) {} - void setVideoInput(QVideoInputDevice *vid) { video_input = vid ; } + void setVideoInput(const QVideoInputDevice *vid) { video_input = vid ; } virtual QString displayName(int) const { return tr("Required bandwidth") ;} @@ -72,22 +70,15 @@ public: } virtual void getValues(std::map& vals) const - { - RsVOIPDataChunk chunk ; - uint32_t total_size = 0 ; + { vals.clear() ; - while(video_input && video_input->getNextEncodedPacket(chunk)) - { - total_size += chunk.size ; - chunk.clear() ; - } - - vals[std::string("bw")] = (float)total_size ; + if(video_input) + vals[std::string("bw")] = video_input->currentBandwidth() ; } private: - QVideoInputDevice *video_input ; + const QVideoInputDevice *video_input ; }; void voipGraph::setVoipSource(voipGraphSource *gs) @@ -109,7 +100,7 @@ voipGraph::voipGraph(QWidget *parent) AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) : ConfigPage(parent, flags) { - std::cerr << "Creating audioInputConfig object" << std::endl; + std::cerr << "Creating audioInputConfig object" << std::endl; /* Invoke the Qt Designer generated object setup routine */ ui.setupUi(this); @@ -121,22 +112,53 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) abSpeech = NULL; qtTick = NULL; - // Create the video pipeline. - // - videoInput = new QVideoInputDevice(this) ; - videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; - - graph_source = new voipGraphSource ; - ui.voipBwGraph->setSource(graph_source); - - graph_source->setVideoInput(videoInput) ; - graph_source->setCollectionTimeLimit(1000*300) ; - graph_source->start() ; + // Create the video pipeline. + // + videoInput = new QVideoInputDevice(this) ; + videoInput->setEchoVideoTarget(ui.videoDisplay) ; + + videoProcessor = new VideoProcessor() ; + videoProcessor->setDisplayTarget(NULL) ; + + videoProcessor->setMaximumBandwidth(ui.availableBW_SB->value()) ; + + videoInput->setVideoProcessor(videoProcessor) ; + + graph_source = new voipGraphSource ; + ui.voipBwGraph->setSource(graph_source); + + graph_source->setVideoInput(videoInput) ; + graph_source->setCollectionTimeLimit(1000*300) ; + graph_source->start() ; + + QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; + QObject::connect(ui.availableBW_SB,SIGNAL(valueChanged(double)),this,SLOT(updateAvailableBW(double))) ; +} + +void AudioInputConfig::updateAvailableBW(double r) +{ + std::cerr << "Setting max bandwidth to " << r << " KB/s" << std::endl; + videoProcessor->setMaximumBandwidth((uint32_t)(r*1024)) ; +} + +void AudioInputConfig::togglePreview(bool b) +{ + if(b) + { + videoInput->setEchoVideoTarget(NULL) ; + videoProcessor->setDisplayTarget(ui.videoDisplay) ; + } + else + { + videoInput->setEchoVideoTarget(ui.videoDisplay) ; + videoProcessor->setDisplayTarget(NULL) ; + } } AudioInputConfig::~AudioInputConfig() { + disconnect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) ); + graph_source->stop() ; graph_source->setVideoInput(NULL) ; @@ -166,7 +188,7 @@ void AudioInputConfig::load() //connect( ui.allowIpDeterminationCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleIpDetermination(bool) ) ); //connect( ui.allowTunnelConnectionCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleTunnelConnection(bool) ) ); - qtTick = new QTimer(this); + qtTick = new RsProtectedTimer(this); connect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) ); qtTick->start(20); /*if (AudioInputRegistrar::qmNew) { @@ -330,8 +352,10 @@ void AudioInputConfig::on_qcbTransmit_currentIndexChanged(int v) { } -void AudioInputConfig::on_Tick_timeout() { - if (!inputAudioProcessor) { +void AudioInputConfig::on_Tick_timeout() +{ + if (!inputAudioProcessor) + { inputAudioProcessor = new QtSpeex::SpeexInputProcessor(); inputAudioProcessor->open(QIODevice::WriteOnly | QIODevice::Unbuffered); @@ -352,6 +376,15 @@ void AudioInputConfig::on_Tick_timeout() { abSpeech->iValue = iroundf(inputAudioProcessor->dVoiceAcivityLevel * 32767.0f + 0.5f); abSpeech->update(); + + // also transmit encoded video + RsVOIPDataChunk chunk ; + + while((!videoInput->stopped()) && videoInput->getNextEncodedPacket(chunk)) + { + videoProcessor->receiveEncodedData(chunk) ; + chunk.clear() ; + } } void AudioInputConfig::emptyBuffer() { diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index cc0fd15c3..ef53c28e9 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -69,6 +69,7 @@ class AudioInputConfig : public ConfigPage //VideoDecoder *videoDecoder ; //VideoEncoder *videoEncoder ; QVideoInputDevice *videoInput ; + VideoProcessor *videoProcessor ; bool loaded; voipGraphSource *graph_source ; @@ -92,10 +93,12 @@ class AudioInputConfig : public ConfigPage virtual QPixmap iconPixmap() const { return QPixmap(":/images/talking_on.svg") ; } virtual QString pageName() const { return tr("VOIP") ; } virtual QString helpText() const { return ""; } - - private slots: + +private slots: + void updateAvailableBW(double r); void loadSettings(); void emptyBuffer(); + void togglePreview(bool) ; void on_qsTransmitHold_valueChanged(int v); void on_qsAmp_valueChanged(int v); diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 8037dea93..62e8c98a6 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -7,7 +7,7 @@ 0 0 1155 - 713 + 832 @@ -391,6 +391,49 @@ + + + + + + Available bandwidth: + + + + + + + <html><head/><body><p>Use this field to simulate the maximum bandwidth available so as to preview what the encoded video will look like with the corresponding compression rate.</p></body></html> + + + KB/s + + + 1 + + + 2.000000000000000 + + + 200.000000000000000 + + + 30.000000000000000 + + + + + + + + + <html><head/><body><p>Display encoded (and then decoded) frame, to check the codec's quality. If not selected, the image above only shows the frame that is grabbed from your camera.</p></body></html> + + + preview + + + diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index eaec96f71..b81b7b79c 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -11,10 +11,15 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) { _timer = NULL ; _capture_device = NULL ; - _video_encoder = NULL ; + _video_processor = NULL ; _echo_output_device = NULL ; } +bool QVideoInputDevice::stopped() +{ + return _timer == NULL ; +} + void QVideoInputDevice::stop() { if(_timer != NULL) @@ -54,49 +59,60 @@ void QVideoInputDevice::start() void QVideoInputDevice::grabFrame() { - IplImage *img=cvQueryFrame(_capture_device); + if(!_timer) + return ; + + IplImage *img=cvQueryFrame(_capture_device); - if(img == NULL) - { - std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; - return ; - } - // get the image data + if(img == NULL) + { + std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; + return ; + } + // get the image data - if(img->nChannels != 3) - { - std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl; - return ; - } + if(img->nChannels != 3) + { + std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl; + return ; + } // convert to RGB and copy to new buffer, because cvQueryFrame tells us to not modify the buffer cv::Mat img_rgb; cv::cvtColor(cv::Mat(img), img_rgb, CV_BGR2RGB); - static const int _encoded_width = 128 ; - static const int _encoded_height = 128 ; + QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888); - QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888).scaled(QSize(_encoded_width,_encoded_height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + if(_video_processor != NULL) + { + _video_processor->processImage(image) ; - if(_video_encoder != NULL) - { - _video_encoder->addImage(image) ; - emit networkPacketReady() ; - } - if(_echo_output_device != NULL) _echo_output_device->showFrame(image) ; + emit networkPacketReady() ; + } + if(_echo_output_device != NULL) + _echo_output_device->showFrame(image) ; } bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { - if(_video_encoder) - return _video_encoder->nextPacket(chunk) ; + if(!_timer) + return false ; + + if(_video_processor) + return _video_processor->nextEncodedPacket(chunk) ; else - return false ; + return false ; +} + +uint32_t QVideoInputDevice::currentBandwidth() const +{ + return _video_processor->currentBandwidthOut() ; } QVideoInputDevice::~QVideoInputDevice() { - stop() ; + stop() ; + _video_processor = NULL ; } @@ -113,6 +129,7 @@ void QVideoOutputDevice::showFrameOff() void QVideoOutputDevice::showFrame(const QImage& img) { - setPixmap(QPixmap::fromImage(img).scaled(minimumSize(),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; + std::cerr << "img.size = " << img.width() << " x " << img.height() << std::endl; + setPixmap(QPixmap::fromImage(img).scaled( QSize(height()*640/480,height()),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; } diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index 7cb62ca0c..cc92302a2 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -2,6 +2,7 @@ #include #include "interface/rsVOIP.h" +#include "gui/VideoProcessor.h" class VideoEncoder ; class CvCapture ; @@ -31,7 +32,7 @@ class QVideoInputDevice: public QObject // Captured images are sent to this encoder. Can be NULL. // - void setVideoEncoder(VideoEncoder *venc) { _video_encoder = venc ; } + void setVideoProcessor(VideoProcessor *venc) { _video_processor = venc ; } // All images received will be echoed to this target. We could use signal/slots, but it's // probably faster this way. Can be NULL. @@ -42,17 +43,23 @@ class QVideoInputDevice: public QObject // bool getNextEncodedPacket(RsVOIPDataChunk&) ; + // gets the estimated current bandwidth required to transmit the encoded data, in B/s + // + uint32_t currentBandwidth() const ; + + // control + void start() ; void stop() ; - - protected slots: + bool stopped(); +protected slots: void grabFrame() ; signals: void networkPacketReady() ; private: - VideoEncoder *_video_encoder ; + VideoProcessor *_video_processor ; QTimer *_timer ; CvCapture *_capture_device ; diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp index 1d94960d7..f8760132d 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp @@ -123,8 +123,7 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n inputAudioDevice = NULL ; inputVideoDevice = new QVideoInputDevice(mChatWidget) ; // not started yet ;-) - inputVideoProcessor = new JPEGVideoEncoder ; - outputVideoProcessor = new JPEGVideoDecoder ; + videoProcessor = new VideoProcessor ; // Make a widget with two video devices, one for echo, and one for the talking peer. videoWidget = new QWidget(mChatWidget) ; @@ -144,8 +143,8 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n mChatWidget->addChatHorizontalWidget(videoWidget) ; inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ; - inputVideoDevice->setVideoEncoder(inputVideoProcessor) ; - outputVideoProcessor->setDisplayTarget(outputVideoDevice) ; + inputVideoDevice->setVideoProcessor(videoProcessor) ; + videoProcessor->setDisplayTarget(outputVideoDevice) ; } VOIPChatWidgetHolder::~VOIPChatWidgetHolder() @@ -154,8 +153,7 @@ VOIPChatWidgetHolder::~VOIPChatWidgetHolder() inputAudioDevice->stop() ; delete inputVideoDevice ; - delete inputVideoProcessor ; - delete outputVideoProcessor ; + delete videoProcessor ; button_map::iterator it = buttonMapTakeVideo.begin(); while (it != buttonMapTakeVideo.end()) { @@ -287,42 +285,50 @@ void VOIPChatWidgetHolder::toggleVideoCapture() void VOIPChatWidgetHolder::addVideoData(const RsPeerId &peer_id, QByteArray* array) { - if (!videoCaptureToggleButton->isChecked()) { - if (mChatWidget) { - QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str()); - if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId - button_map::iterator it = buttonMapTakeVideo.find(buttonName); - if (it == buttonMapTakeVideo.end()){ - mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime() - , tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM); - RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call")); - button->setToolTip(tr("Activate camera")); - button->setStyleSheet(QString("border: 1px solid #199909;") - .append("font-size: 12pt; color: white;") - .append("min-width: 128px; min-height: 24px;") - .append("border-radius: 6px;") - .append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, " - "stop: 0 #22c70d, stop: 1 #116a06);") + if (!videoCaptureToggleButton->isChecked()) + { + if (mChatWidget) { + QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str()); + if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId + button_map::iterator it = buttonMapTakeVideo.find(buttonName); + if (it == buttonMapTakeVideo.end()){ + mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime() + , tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM); + RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call")); + button->setToolTip(tr("Activate camera")); + button->setStyleSheet(QString("border: 1px solid #199909;") + .append("font-size: 12pt; color: white;") + .append("min-width: 128px; min-height: 24px;") + .append("border-radius: 6px;") + .append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, " + "stop: 0 #22c70d, stop: 1 #116a06);") - ); + ); - button->updateImage(); + button->updateImage(); - connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture())); - connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter())); - connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave())); + connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture())); + connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter())); + connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave())); - buttonMapTakeVideo.insert(buttonName, button); - } - } + buttonMapTakeVideo.insert(buttonName, button); + } + } - //TODO make a sound for the incoming call -// soundManager->play(VOIP_SOUND_INCOMING_CALL); - if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id); + //TODO make a sound for the incoming call + // soundManager->play(VOIP_SOUND_INCOMING_CALL); + if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id); - } else { - outputVideoProcessor->receiveEncodedData((unsigned char *)array->data(),array->size()) ; - } + } + else + { + RsVOIPDataChunk chunk ; + chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + chunk.size = array->size() ; + chunk.data = array->data() ; + + videoProcessor->receiveEncodedData(chunk) ; + } } void VOIPChatWidgetHolder::botMouseEnter() @@ -359,7 +365,7 @@ void VOIPChatWidgetHolder::botMouseLeave() void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec) { - inputVideoProcessor->setMaximumFrameRate(bytes_per_sec) ; + videoProcessor->setMaximumBandwidth(bytes_per_sec) ; } void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array) diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.h b/plugins/VOIP/gui/VOIPChatWidgetHolder.h index 791a2dcf8..350f49f2c 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.h +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.h @@ -34,8 +34,7 @@ class QAudioInput; class QAudioOutput; class QVideoInputDevice ; class QVideoOutputDevice ; -class VideoEncoder ; -class VideoDecoder ; +class VideoProcessor ; #define VOIP_SOUND_INCOMING_CALL "VOIP_incoming_call" @@ -82,8 +81,7 @@ protected: QWidget *videoWidget ; // pointer to call show/hide - VideoEncoder *inputVideoProcessor; - VideoDecoder *outputVideoProcessor; + VideoProcessor *videoProcessor; // Additional buttons to the chat bar QToolButton *audioListenToggleButton ; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 27343c412..a0084ea18 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include @@ -7,67 +9,673 @@ #include "VideoProcessor.h" #include "QVideoDevice.h" -VideoDecoder::VideoDecoder() +#include + + +extern "C" { +#include + +#include +#include +#include +#include +#include +#include +} +//#define DEBUG_MPEG_VIDEO 1 + +VideoProcessor::VideoProcessor() + :_encoded_frame_size(640,480) , vpMtx("VideoProcessor") { - _output_device = NULL ; + _decoded_output_device = NULL ; + + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; + _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; + + _estimated_bandwidth_in = 0 ; + _estimated_bandwidth_out = 0 ; + _target_bandwidth_out = 30*1024 ; // 30 KB/s + + _total_encoded_size_in = 0 ; + _total_encoded_size_out = 0 ; + + _last_bw_estimate_in_TS = time(NULL) ; + _last_bw_estimate_out_TS = time(NULL) ; } -bool VideoEncoder::addImage(const QImage& img) +VideoProcessor::~VideoProcessor() { - encodeData(img) ; - - return true ; + // clear encoding queue + + RS_STACK_MUTEX(vpMtx) ; + + while(!_encoded_out_queue.empty()) + { + _encoded_out_queue.back().clear() ; + _encoded_out_queue.pop_back() ; + } } -bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk) +bool VideoProcessor::processImage(const QImage& img) { - if(_out_queue.empty()) + VideoCodec *codec ; + + switch(_encoding_current_codec) + { + case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; + break ; + case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; + break ; + default: + codec = NULL ; + } + + // std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; + + if(codec) + { + RsVOIPDataChunk chunk ; + + if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),_target_bandwidth_out,chunk) && chunk.size > 0) + { + RS_STACK_MUTEX(vpMtx) ; + _encoded_out_queue.push_back(chunk) ; + _total_encoded_size_out += chunk.size ; + } + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_out_TS) + { + RS_STACK_MUTEX(vpMtx) ; + + _estimated_bandwidth_out = uint32_t(0.75*_estimated_bandwidth_out + 0.25 * (_total_encoded_size_out / (float)(now - _last_bw_estimate_out_TS))) ; + + _total_encoded_size_out = 0 ; + _last_bw_estimate_out_TS = now ; + +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE + std::cerr << "new bw estimate: " << _estimated_bw << std::endl; +#endif + } + + return true ; + } + else + { + std::cerr << "No codec for codec ID = " << _encoding_current_codec << ". Please call VideoProcessor::setCurrentCodec()" << std::endl; + return false ; + } +} + +bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) +{ + RS_STACK_MUTEX(vpMtx) ; + if(_encoded_out_queue.empty()) return false ; - chunk = _out_queue.front() ; - _out_queue.pop_front() ; + chunk = _encoded_out_queue.front() ; + _encoded_out_queue.pop_front() ; return true ; } -void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) +void VideoProcessor::setInternalFrameSize(QSize s) { - _output_device->showFrame(decodeData(data,size)) ; + _encoded_frame_size = s ; } -QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint32_t size) +void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) { - QByteArray qb((char*)encoded_image_data,size) ; - QImage image ; - if(image.loadFromData(qb,"JPEG")) - return image ; + static const int HEADER_SIZE = 4 ; + + // read frame type. Use first 4 bytes to give info about content. + // + // Byte Meaning Values + // 00 Codec CODEC_ID_JPEG_VIDEO Basic Jpeg codec + // CODEC_ID_DDWT_VIDEO Differential wavelet compression + // + // 01 Unused Might be useful later + // + // 0203 Flags Codec specific flags. + // + + if(chunk.size < HEADER_SIZE) + { + std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << chunk.size << std::endl; + return ; + } + + uint32_t codid = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; + //uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; + + VideoCodec *codec ; + + switch(codid) + { + case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; + break ; + case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; + break ; + default: + codec = NULL ; + } + QImage img ; + + if(codec == NULL) + { + std::cerr << "Unknown decoding codec: " << codid << std::endl; + return ; + } + + { + RS_STACK_MUTEX(vpMtx) ; + _total_encoded_size_in += chunk.size ; + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_in_TS) + { + _estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ; + + _total_encoded_size_in = 0 ; + _last_bw_estimate_in_TS = now ; + +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE + std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; +#endif + } + } + if(!codec->decodeData(chunk,img)) + { + std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; + return ; + } + + if(_decoded_output_device) + _decoded_output_device->showFrame(img) ; +} + +void VideoProcessor::setMaximumBandwidth(uint32_t bytes_per_sec) +{ + std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; + _target_bandwidth_out = bytes_per_sec ; +} + +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +JPEGVideo::JPEGVideo() + : _encoded_ref_frame_max_distance(10),_encoded_ref_frame_count(10) +{ +} + +bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) +{ + // now see if the frame is a differential frame, or just a reference frame. + + uint16_t codec = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; + uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; + + assert(codec == VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO) ; + + // un-compress image data + + QByteArray qb((char*)&((uint8_t*)chunk.data)[HEADER_SIZE],(int)chunk.size - HEADER_SIZE) ; + + if(!image.loadFromData(qb,"JPEG")) + { + std::cerr << "image.loadFromData(): returned an error.: " << std::endl; + return false ; + } + + if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) + { + if(_decoded_reference_frame.size() != image.size()) + { + std::cerr << "Bad reference frame!" << std::endl; + return false ; + } + + QImage res = _decoded_reference_frame ; + + for(int i=0;i> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],qb.data(),qb.size()) ; + + voip_chunk.size = HEADER_SIZE + qb.size() ; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + return true ; +} + +FFmpegVideo::FFmpegVideo() +{ + // Encoding + + encoding_codec = NULL ; + encoding_frame_buffer = NULL ; + encoding_context = NULL ; + + //AVCodecID codec_id = AV_CODEC_ID_H264 ; + //AVCodecID codec_id = AV_CODEC_ID_MPEG2VIDEO; + AVCodecID codec_id = AV_CODEC_ID_MPEG4; + + uint8_t endcode[] = { 0, 0, 1, 0xb7 }; + + /* find the mpeg1 video encoder */ + encoding_codec = avcodec_find_encoder(codec_id); + + if (!encoding_codec) throw("AV codec not found for codec id ") ; + + encoding_context = avcodec_alloc_context3(encoding_codec); + + if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); + + /* put sample parameters */ + encoding_context->bit_rate = 10*1024 ; // default bitrate is 30KB/s + encoding_context->bit_rate_tolerance = encoding_context->bit_rate ; + +#ifdef USE_VARIABLE_BITRATE + encoding_context->rc_min_rate = 0; + encoding_context->rc_max_rate = 10*1024;//encoding_context->bit_rate; + encoding_context->rc_buffer_size = 10*1024*1024; + encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size); + encoding_context->rc_max_available_vbv_use = 1.0; + encoding_context->rc_min_vbv_overflow_use = 0.0; +#else + encoding_context->rc_min_rate = 0; + encoding_context->rc_max_rate = 0; + encoding_context->rc_buffer_size = 0; +#endif + encoding_context->flags |= CODEC_FLAG_PSNR; + encoding_context->flags |= CODEC_FLAG_TRUNCATED; + encoding_context->flags |= CODEC_CAP_PARAM_CHANGE; + encoding_context->i_quant_factor = 0.769f; + encoding_context->b_quant_factor = 1.4f; + encoding_context->time_base.num = 1; + encoding_context->time_base.den = 15;//framesPerSecond; + encoding_context->qmin = 1; + encoding_context->qmax = 51; + encoding_context->max_qdiff = 4; + + //encoding_context->me_method = ME_HEX; + //encoding_context->max_b_frames = 4; + //encoding_context->flags |= CODEC_FLAG_LOW_DELAY; // MPEG2 only + //encoding_context->partitions = X264_PART_I4X4 | X264_PART_I8X8 | X264_PART_P8X8 | X264_PART_P4X4 | X264_PART_B8X8; + //encoding_context->crf = 0.0f; + //encoding_context->cqp = 26; + + /* resolution must be a multiple of two */ + encoding_context->width = 640;//176; + encoding_context->height = 480;//144; + /* frames per second */ + encoding_context->time_base = (AVRational){1,25}; + /* emit one intra frame every ten frames + * check frame pict_type before passing frame + * to encoder, if frame->pict_type is AV_PICTURE_TYPE_I + * then gop_size is ignored and the output of encoder + * will always be I frame irrespective to gop_size + */ + encoding_context->gop_size = 100; + //encoding_context->max_b_frames = 1; + encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; //context->pix_fmt = AV_PIX_FMT_RGB24; + + if (codec_id == AV_CODEC_ID_H264) + av_opt_set(encoding_context->priv_data, "preset", "slow", 0); + + /* open it */ + if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0) + throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); + + encoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + + if(!encoding_frame_buffer) + throw std::runtime_error("AV: could not allocate frame buffer.") ; + + encoding_frame_buffer->format = encoding_context->pix_fmt; + encoding_frame_buffer->width = encoding_context->width; + encoding_frame_buffer->height = encoding_context->height; + + /* the image can be allocated by any means and av_image_alloc() is + * just the most convenient way if av_malloc() is to be used */ + + int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, + encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); + + if (ret < 0) + throw std::runtime_error("AV: Could not allocate raw picture buffer"); + + encoding_frame_count = 0 ; + + // Decoding + + decoding_codec = avcodec_find_decoder(codec_id); + + if (!decoding_codec) + throw("AV codec not found for codec id ") ; + + decoding_context = avcodec_alloc_context3(decoding_codec); + + if(!decoding_context) + throw std::runtime_error("AV: Could not allocate video codec decoding context"); + + decoding_context->width = encoding_context->width; + decoding_context->height = encoding_context->height; + decoding_context->pix_fmt = AV_PIX_FMT_YUV420P; + + if(decoding_codec->capabilities & CODEC_CAP_TRUNCATED) + decoding_context->flags |= CODEC_FLAG_TRUNCATED; // we do not send complete frames + + if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) + throw("AV codec open action failed! ") ; + + decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + + av_init_packet(&decoding_buffer); + decoding_buffer.data = NULL ; + decoding_buffer.size = 0 ; + + //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); + + //if (ret < 0) + //throw std::runtime_error("AV: Could not allocate raw picture buffer"); + + // debug +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Dumping captured data to file tmpvideo.mpg" << std::endl; + encoding_debug_file = fopen("tmpvideo.mpg","w") ; +#endif +} + +FFmpegVideo::~FFmpegVideo() +{ + avcodec_close(encoding_context); + avcodec_close(decoding_context); + av_free(encoding_context); + av_free(decoding_context); + av_freep(&encoding_frame_buffer->data[0]); + av_freep(&decoding_frame_buffer->data[0]); + free(encoding_frame_buffer); + free(decoding_frame_buffer); +} + +#define MAX_FFMPEG_ENCODING_BITRATE 81920 + +bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) +{ +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; +#endif + QImage input ; + + if(target_encoding_bitrate > MAX_FFMPEG_ENCODING_BITRATE) + { + std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl; + target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ; + } + //encoding_context->bit_rate = target_encoding_bitrate; + encoding_context->rc_max_rate = target_encoding_bitrate; + //encoding_context->bit_rate_tolerance = target_encoding_bitrate; + + if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) + input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + else + input = image ; + + /* prepare a dummy image */ + /* Y */ + for (int y = 0; y < encoding_context->height/2; y++) + for (int x = 0; x < encoding_context->width/2; x++) + { + QRgb pix00 = input.pixel(QPoint(2*x+0,2*y+0)) ; + QRgb pix01 = input.pixel(QPoint(2*x+0,2*y+1)) ; + QRgb pix10 = input.pixel(QPoint(2*x+1,2*y+0)) ; + QRgb pix11 = input.pixel(QPoint(2*x+1,2*y+1)) ; + + int R00 = (pix00 >> 16) & 0xff ; int G00 = (pix00 >> 8) & 0xff ; int B00 = (pix00 >> 0) & 0xff ; + int R01 = (pix01 >> 16) & 0xff ; int G01 = (pix01 >> 8) & 0xff ; int B01 = (pix01 >> 0) & 0xff ; + int R10 = (pix10 >> 16) & 0xff ; int G10 = (pix10 >> 8) & 0xff ; int B10 = (pix10 >> 0) & 0xff ; + int R11 = (pix11 >> 16) & 0xff ; int G11 = (pix11 >> 8) & 0xff ; int B11 = (pix11 >> 0) & 0xff ; + + int Y00 = (0.257 * R00) + (0.504 * G00) + (0.098 * B00) + 16 ; + int Y01 = (0.257 * R01) + (0.504 * G01) + (0.098 * B01) + 16 ; + int Y10 = (0.257 * R10) + (0.504 * G10) + (0.098 * B10) + 16 ; + int Y11 = (0.257 * R11) + (0.504 * G11) + (0.098 * B11) + 16 ; + + float R = 0.25*(R00+R01+R10+R11) ; + float G = 0.25*(G00+G01+G10+G11) ; + float B = 0.25*(B00+B01+B10+B11) ; + + int U = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; + int V = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; + + encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y00)); // Y + encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y01)); // Y + encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y10)); // Y + encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y11)); // Y + + encoding_frame_buffer->data[1][y * encoding_frame_buffer->linesize[1] + x] = std::min(255,std::max(0,U));// Cr + encoding_frame_buffer->data[2][y * encoding_frame_buffer->linesize[2] + x] = std::min(255,std::max(0,V));// Cb + } + + + encoding_frame_buffer->pts = encoding_frame_count++; + + /* encode the image */ + + int got_output = 0; + + AVFrame *frame = encoding_frame_buffer ; + + AVPacket pkt ; + av_init_packet(&pkt); + pkt.data = NULL; // packet data will be allocated by the encoder + pkt.size = 0; + + // do + // { + int ret = avcodec_encode_video2(encoding_context, &pkt, frame, &got_output) ; + + if (ret < 0) + { + std::cerr << "Error encoding frame!" << std::endl; + return false ; + } + // frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer + // + // } while(got_output) ; + + if(got_output) + { + voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; + uint32_t flags = 0; + + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ; + + voip_chunk.size = pkt.size + HEADER_SIZE; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; + fflush(encoding_debug_file) ; +#endif + av_free_packet(&pkt); + + return true ; + } else { - std::cerr << "image.loadFromData(): returned an error.: " << std::endl; - return QImage() ; + voip_chunk.data = NULL; + voip_chunk.size = 0; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "No output produced." << std::endl; + return false ; } + } -void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec) +bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { - std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; -} - - -void JPEGVideoEncoder::encodeData(const QImage& image) -{ - QByteArray qb ; - - QBuffer buffer(&qb) ; - buffer.open(QIODevice::WriteOnly) ; - image.save(&buffer,"JPEG") ; - - RsVOIPDataChunk voip_chunk ; - voip_chunk.data = malloc(qb.size()); - memcpy(voip_chunk.data,qb.data(),qb.size()) ; - voip_chunk.size = qb.size() ; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - _out_queue.push_back(voip_chunk) ; +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Decoding data of size " << chunk.size << std::endl; +#endif + unsigned char *buff ; + + if(decoding_buffer.data != NULL) + { +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Completing buffer with size " << chunk.size - HEADER_SIZE + decoding_buffer.size << ": copying existing " + << decoding_buffer.size << " bytes. Adding new " << chunk.size - HEADER_SIZE<< " bytes " << std::endl; +#endif + + uint32_t s = chunk.size - HEADER_SIZE + decoding_buffer.size ; + unsigned char *tmp = (unsigned char*)memalign(16,s + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(tmp,0,s+FF_INPUT_BUFFER_PADDING_SIZE) ; + + memcpy(tmp,decoding_buffer.data,decoding_buffer.size) ; + + free(decoding_buffer.data) ; + + buff = &tmp[decoding_buffer.size] ; + decoding_buffer.size = s ; + decoding_buffer.data = tmp ; + } + else + { +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl; +#endif + + decoding_buffer.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; + decoding_buffer.size = chunk.size - HEADER_SIZE ; + memset(decoding_buffer.data,0,decoding_buffer.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + + buff = decoding_buffer.data ; + } + + memcpy(buff,&((unsigned char*)chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; + + if(len < 0) + { + std::cerr << "Error decodign frame!" << std::endl; + return false ; + } +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Used " << len << " bytes out of " << decoding_buffer.size << ". got_frame = " << got_frame << std::endl; +#endif + + if(got_frame) + { + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; +#endif + + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ; + int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ; + + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + } + + if(len == decoding_buffer.size) + { + free(decoding_buffer.data) ; + decoding_buffer.data = NULL; + decoding_buffer.size = 0; + } + else if(len != 0) + { +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Moving remaining data (" << decoding_buffer.size - len << " bytes) back to 0" << std::endl; +#endif + + memmove(decoding_buffer.data,decoding_buffer.data+len,decoding_buffer.size - len) ; + decoding_buffer.size -= len ; + } + + return true ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 56894bcea..a3869dc5e 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -4,87 +4,149 @@ #include #include "interface/rsVOIP.h" +extern "C" { +#include +} + class QVideoOutputDevice ; -// This class decodes video from a stream. It keeps a queue of -// decoded frame that needs to be retrieved using the getNextImage() method. -// -class VideoDecoder +class VideoCodec { - public: - VideoDecoder() ; - virtual ~VideoDecoder() {} - - // Gets the next image to be displayed. Once returned, the image should - // be cleared from the incoming queue. - // - void setDisplayTarget(QVideoOutputDevice *odev) { _output_device = odev ; } - - virtual void receiveEncodedData(const unsigned char *data,uint32_t size) ; - - // returns the current (measured) frame rate in bytes per second. - // - uint32_t currentFrameRate() const; - - private: - QVideoOutputDevice *_output_device ; - - std::list _image_queue ; - - // Incoming data is processed by a video codec and converted into images. - // - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) = 0 ; - -// // This buffer accumulated incoming encoded data, until a full packet is obtained, -// // since the stream might not send images at once. When incoming images are decoded, the -// // data is removed from the buffer. -// // -// unsigned char *buffer ; -// uint32_t buffer_size ; -}; - -// This class encodes video using a video codec (possibly homemade, or based on existing codecs) -// and produces a data stream that is sent to the network transfer service (e.g. p3VOIP). -// -class VideoEncoder -{ - public: - VideoEncoder() {} - virtual ~VideoEncoder() {} - - // Takes the next image to be encoded. - // - bool addImage(const QImage& Image) ; - - bool packetReady() const { return !_out_queue.empty() ; } - bool nextPacket(RsVOIPDataChunk& ) ; - - // Used to tweak the compression ratio so that the video can stream ok. - // - void setMaximumFrameRate(uint32_t bytes_per_second) ; - - protected: - //virtual bool sendEncodedData(unsigned char *mem,uint32_t size) = 0 ; - virtual void encodeData(const QImage& image) = 0 ; - - std::list _out_queue ; +public: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) = 0; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) = 0; + +protected: + static const uint32_t HEADER_SIZE = 0x04 ; }; // Now derive various image encoding/decoding algorithms. // -class JPEGVideoDecoder: public VideoDecoder +class JPEGVideo: public VideoCodec { - protected: - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; +public: + JPEGVideo() ; + +protected: + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; + + static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; +private: + QImage _decoded_reference_frame ; + QImage _encoded_reference_frame ; + + uint32_t _encoded_ref_frame_max_distance ; // max distance between two reference frames. + uint32_t _encoded_ref_frame_count ; }; -class JPEGVideoEncoder: public VideoEncoder +struct AVCodec ; +struct AVCodecContext ; +struct AVFrame ; +struct AVPacket ; + +class FFmpegVideo: public VideoCodec +{ +public: + FFmpegVideo() ; + ~FFmpegVideo() ; + +protected: + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; + +private: + AVCodec *encoding_codec; + AVCodec *decoding_codec; + AVCodecContext *encoding_context; + AVCodecContext *decoding_context; + AVFrame *encoding_frame_buffer ; + AVFrame *decoding_frame_buffer ; + AVPacket decoding_buffer; + uint64_t encoding_frame_count ; + + FILE *encoding_debug_file ; +}; + +// This class decodes video from a stream. It keeps a queue of +// decoded frame that needs to be retrieved using the getNextImage() method. +// +class VideoProcessor { public: - JPEGVideoEncoder() {} + VideoProcessor() ; + virtual ~VideoProcessor() ; + enum CodecId { + VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000, + VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO = 0x0001, + VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002, + VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO = 0x0003 + }; + +// ===================================================================================== +// =------------------------------------ DECODING -------------------------------------= +// ===================================================================================== + + // Gets the next image to be displayed. Once returned, the image should + // be cleared from the incoming queue. + // + void setDisplayTarget(QVideoOutputDevice *odev) { _decoded_output_device = odev ; } + virtual void receiveEncodedData(const RsVOIPDataChunk& chunk) ; + + // returns the current (measured) frame rate in bytes per second. + // + uint32_t currentBandwidthIn() const { return _estimated_bandwidth_in ; } + + private: + QVideoOutputDevice *_decoded_output_device ; + std::list _decoded_image_queue ; + +// ===================================================================================== +// =------------------------------------ ENCODING -------------------------------------= +// ===================================================================================== + + public: + // Takes the next image to be encoded. + // + bool processImage(const QImage& Image) ; + bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; } + bool nextEncodedPacket(RsVOIPDataChunk& ) ; + + // Used to tweak the compression ratio so that the video can stream ok. + // + void setMaximumBandwidth(uint32_t bytes_per_second) ; + void setInternalFrameSize(QSize) ; + + // returns the current encoding frame rate in bytes per second. + // + uint32_t currentBandwidthOut() const { return _estimated_bandwidth_out ; } + protected: - virtual void encodeData(const QImage& Image) ; + std::list _encoded_out_queue ; + QSize _encoded_frame_size ; + +// ===================================================================================== +// =------------------------------------- Codecs --------------------------------------= +// ===================================================================================== + + JPEGVideo _jpeg_video_codec ; + FFmpegVideo _mpeg_video_codec ; + + uint16_t _encoding_current_codec ; + + time_t _last_bw_estimate_in_TS; + time_t _last_bw_estimate_out_TS; + + uint32_t _total_encoded_size_in ; + uint32_t _total_encoded_size_out ; + + float _estimated_bandwidth_in ; + float _estimated_bandwidth_out ; + + float _target_bandwidth_out ; + + RsMutex vpMtx ; }; diff --git a/plugins/VOIP/interface/rsVOIP.h b/plugins/VOIP/interface/rsVOIP.h index 7269610b4..25fd0cfbd 100644 --- a/plugins/VOIP/interface/rsVOIP.h +++ b/plugins/VOIP/interface/rsVOIP.h @@ -51,7 +51,9 @@ class RsVOIPPongResult struct RsVOIPDataChunk { - typedef enum { RS_VOIP_DATA_TYPE_AUDIO, RS_VOIP_DATA_TYPE_VIDEO } RsVOIPDataType ; + typedef enum { RS_VOIP_DATA_TYPE_UNKNOWN = 0x00, + RS_VOIP_DATA_TYPE_AUDIO = 0x01, + RS_VOIP_DATA_TYPE_VIDEO = 0x02 } RsVOIPDataType ; void *data ; // create/delete using malloc/free. uint32_t size ; diff --git a/plugins/VOIP/services/rsVOIPItems.cc b/plugins/VOIP/services/rsVOIPItems.cc index eb8842468..a75fa4c16 100644 --- a/plugins/VOIP/services/rsVOIPItems.cc +++ b/plugins/VOIP/services/rsVOIPItems.cc @@ -76,6 +76,19 @@ std::ostream& RsVOIPProtocolItem::print(std::ostream &out, uint16_t indent) printRsItemEnd(out, "RsVOIPProtocolItem", indent); return out; } +std::ostream& RsVOIPBandwidthItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsVOIPBandwidthItem", indent); + uint16_t int_Indent = indent + 2; + printIndent(out, int_Indent); + out << "flags: " << std::hex << flags << std::dec << std::endl; + + printIndent(out, int_Indent); + out << "speed: " << bytes_per_sec << std::endl; + + printRsItemEnd(out, "RsVOIPBandwidthItem", indent); + return out; +} std::ostream& RsVOIPDataItem::print(std::ostream &out, uint16_t indent) { printRsItemBase(out, "RsVOIPDataItem", indent); @@ -100,6 +113,14 @@ uint32_t RsVOIPDataItem::serial_size() const return s; } +uint32_t RsVOIPBandwidthItem::serial_size() const +{ + uint32_t s = 8; /* header */ + s += 4; /* flags */ + s += 4; /* bandwidth */ + + return s; +} uint32_t RsVOIPProtocolItem::serial_size() const { uint32_t s = 8; /* header */ @@ -150,6 +171,40 @@ bool RsVOIPProtocolItem::serialise(void *data, uint32_t& pktsize) return ok; } +bool RsVOIPBandwidthItem::serialise(void *data, uint32_t& pktsize) +{ + uint32_t tlvsize = serial_size() ; + uint32_t offset = 0; + + if (pktsize < tlvsize) + return false; /* not enough space */ + + pktsize = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, PacketId(), tlvsize); + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Header: " << ok << std::endl; + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size: " << tlvsize << std::endl; +#endif + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= setRawUInt32(data, tlvsize, &offset, flags); + ok &= setRawUInt32(data, tlvsize, &offset, bytes_per_sec); + + if (offset != tlvsize) + { + ok = false; + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size Error! " << std::endl; + } + + return ok; +} /* serialise the data to the buffer */ bool RsVOIPDataItem::serialise(void *data, uint32_t& pktsize) { @@ -254,6 +309,36 @@ RsVOIPProtocolItem::RsVOIPProtocolItem(void *data, uint32_t pktsize) if (!ok) throw std::runtime_error("Deserialisation error!") ; } +RsVOIPBandwidthItem::RsVOIPBandwidthItem(void *data, uint32_t pktsize) + : RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH) +{ + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + uint32_t offset = 0; + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_VOIP_PLUGIN != getRsItemService(rstype)) || (RS_PKT_SUBTYPE_VOIP_BANDWIDTH != getRsItemSubType(rstype))) + throw std::runtime_error("Wrong packet type!") ; + + if (pktsize < rssize) /* check size */ + throw std::runtime_error("Not enough size!") ; + + bool ok = true; + + /* skip the header */ + offset += 8; + + /* get mandatory parts first */ + ok &= getRawUInt32(data, rssize, &offset, &flags); + ok &= getRawUInt32(data, rssize, &offset, &bytes_per_sec); + + if (offset != rssize) + throw std::runtime_error("Deserialisation error!") ; + + if (!ok) + throw std::runtime_error("Deserialisation error!") ; +} RsVOIPPingItem::RsVOIPPingItem(void *data, uint32_t pktsize) : RsVOIPItem(RS_PKT_SUBTYPE_VOIP_PING) { diff --git a/plugins/VOIP/services/rsVOIPItems.h b/plugins/VOIP/services/rsVOIPItems.h index fa440ef19..7658ea4cc 100644 --- a/plugins/VOIP/services/rsVOIPItems.h +++ b/plugins/VOIP/services/rsVOIPItems.h @@ -55,11 +55,12 @@ const uint16_t RS_SERVICE_TYPE_VOIP_PLUGIN = 0xa021; -const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; -const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; -const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ; - // 0x04 is unused because of a change in the protocol -const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x05 ; +const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; +const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; +const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ; + // 0x04,0x05 is unused because of a change in the protocol +const uint8_t RS_PKT_SUBTYPE_VOIP_BANDWIDTH = 0x06 ; +const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x07 ; const uint8_t QOS_PRIORITY_RS_VOIP = 9 ; @@ -117,9 +118,27 @@ class RsVOIPDataItem: public RsVOIPItem uint32_t flags ; uint32_t data_size ; + void *voip_data ; }; +class RsVOIPBandwidthItem: public RsVOIPItem +{ + public: + RsVOIPBandwidthItem() :RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH) {} + RsVOIPBandwidthItem(void *data,uint32_t size) ; // de-serialization + + virtual bool serialise(void *data,uint32_t& size) ; + virtual uint32_t serial_size() const ; + + virtual ~RsVOIPBandwidthItem() {} + virtual std::ostream& print(std::ostream &out, uint16_t indent = 0); + + uint32_t flags ; // is that incoming or expected bandwidth? + uint32_t bytes_per_sec ; // bandwidth in bytes per sec. +}; + + class RsVOIPProtocolItem: public RsVOIPItem { public: