From 94317d1a390f297d182b09f8cddd70c8f6d5c198 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 7 Aug 2015 21:40:48 -0400 Subject: [PATCH 01/43] added graph display of instantly required bandwidth for VOIP, in preparation to chosing new video codec. GUI layout needs to be sorted. --- plugins/VOIP/gui/AudioInputConfig.cpp | 66 ++++++++++++++++++++++- plugins/VOIP/gui/AudioInputConfig.h | 20 ++++++- plugins/VOIP/gui/AudioInputConfig.ui | 76 +++++++++++++++++---------- plugins/VOIP/gui/QVideoDevice.cpp | 5 +- plugins/VOIP/interface/rsVOIP.h | 2 + plugins/VOIP/services/p3VOIP.cc | 8 +++ 6 files changed, 147 insertions(+), 30 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 5c7553ee8..e23083200 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -39,6 +39,7 @@ //#include "NetworkConfig.h" #include "audiodevicehelper.h" #include "AudioWizard.h" +#include "gui/common/RSGraphWidget.h" #include #define iroundf(x) ( static_cast(x) ) @@ -51,6 +52,59 @@ void AudioInputDialog::showEvent(QShowEvent *) { qtTick->start(20); }*/ +class voipGraphSource: public RSGraphSource +{ +public: + voipGraphSource() {} + + void setVideoInput(QVideoInputDevice *vid) { video_input = vid ; } + + virtual QString displayName(int) const { return tr("Required bandwidth") ;} + + virtual QString displayValue(float v) const + { + if(v < 1000) + return QString::number(v,10,2) + " B/s" ; + else if(v < 1000*1024) + return QString::number(v/1024,10,2) + " KB/s" ; + else + return QString::number(v/(1024*1024),10,2) + " MB/s" ; + } + + virtual void getValues(std::map& vals) const + { + RsVOIPDataChunk chunk ; + uint32_t total_size = 0 ; + vals.clear() ; + + while(video_input && video_input->getNextEncodedPacket(chunk)) + { + total_size += chunk.size ; + chunk.clear() ; + } + + vals[std::string("bw")] = (float)total_size ; + } + +private: + QVideoInputDevice *video_input ; +}; + +void voipGraph::setVoipSource(voipGraphSource *gs) +{ +_src = gs ; +RSGraphWidget::setSource(gs) ; +} + +voipGraph::voipGraph(QWidget *parent) + : RSGraphWidget(parent) +{ + setFlags(RSGraphWidget::RSGRAPH_FLAGS_SHOW_LEGEND) ; + setFlags(RSGraphWidget::RSGRAPH_FLAGS_PAINT_STYLE_PLAIN) ; + + _src = NULL ; +} + /** Constructor */ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) : ConfigPage(parent, flags) @@ -71,11 +125,21 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) // videoInput = new QVideoInputDevice(this) ; videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoInput->setVideoEncoder(NULL) ; + videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + + graph_source = new voipGraphSource ; + ui.voipBwGraph->setSource(graph_source); + + graph_source->setVideoInput(videoInput) ; + graph_source->setCollectionTimeLimit(1000*300) ; + graph_source->start() ; } AudioInputConfig::~AudioInputConfig() { + graph_source->stop() ; + graph_source->setVideoInput(NULL) ; + std::cerr << "Deleting audioInputConfig object" << std::endl; if(videoInput != NULL) { diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index 1b64085a3..cc0fd15c3 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -35,10 +35,27 @@ #include "retroshare-gui/configpage.h" -#include "ui_AudioInputConfig.h" #include "SpeexProcessor.h" #include "VideoProcessor.h" #include "AudioStats.h" +#include "gui/common/RSGraphWidget.h" + +class voipGraphSource ; + +class voipGraph: public RSGraphWidget +{ +public: + voipGraph(QWidget *parent) ; + + voipGraphSource *voipSource() const { return _src ; } + + void setVoipSource(voipGraphSource *gs) ; + +private: + voipGraphSource *_src ; +}; + +#include "ui_AudioInputConfig.h" class AudioInputConfig : public ConfigPage { @@ -54,6 +71,7 @@ class AudioInputConfig : public ConfigPage QVideoInputDevice *videoInput ; bool loaded; + voipGraphSource *graph_source ; protected: QTimer *qtTick; diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 6aa071b4a..8037dea93 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -6,8 +6,8 @@ 0 0 - 501 - 406 + 1155 + 713 @@ -348,33 +348,49 @@ - - - - Video Processing - - - - - - - 170 - 128 - - - - QFrame::StyledPanel - - - QFrame::Raised - - - - - - + + + + Video Processing + + + + + + + 170 + 128 + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + + + + + 0 + 0 + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + + + @@ -397,6 +413,12 @@
gui/QVideoDevice.h
1 + + voipGraph + QFrame +
gui/AudioInputConfig.h
+ 1 + qcbTransmit diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index eed14b58b..eaec96f71 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -88,7 +88,10 @@ void QVideoInputDevice::grabFrame() bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { - return _video_encoder->nextPacket(chunk) ; + if(_video_encoder) + return _video_encoder->nextPacket(chunk) ; + else + return false ; } QVideoInputDevice::~QVideoInputDevice() diff --git a/plugins/VOIP/interface/rsVOIP.h b/plugins/VOIP/interface/rsVOIP.h index 18f258769..7269610b4 100644 --- a/plugins/VOIP/interface/rsVOIP.h +++ b/plugins/VOIP/interface/rsVOIP.h @@ -56,6 +56,8 @@ struct RsVOIPDataChunk void *data ; // create/delete using malloc/free. uint32_t size ; RsVOIPDataType type ; // video or audio + + void clear() ; }; class RsVOIP diff --git a/plugins/VOIP/services/p3VOIP.cc b/plugins/VOIP/services/p3VOIP.cc index d7377077c..fb164bd7c 100644 --- a/plugins/VOIP/services/p3VOIP.cc +++ b/plugins/VOIP/services/p3VOIP.cc @@ -147,6 +147,14 @@ RsServiceInfo p3VOIP::getServiceInfo() TURTLE_MIN_MINOR_VERSION); } +void RsVOIPDataChunk::clear() +{ + + if(data) + free(data) ; + data=NULL; + size=0 ; +} int p3VOIP::tick() { #ifdef DEBUG_VOIP From 2107a1c858cca4565435d876056ec5978f6c5999 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 8 Aug 2015 18:46:49 -0400 Subject: [PATCH 02/43] fixed up image display layout in VOIP config tab --- plugins/VOIP/gui/QVideoDevice.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index eaec96f71..67c79ee5b 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -113,6 +113,7 @@ void QVideoOutputDevice::showFrameOff() void QVideoOutputDevice::showFrame(const QImage& img) { - setPixmap(QPixmap::fromImage(img).scaled(minimumSize(),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; + std::cerr << "img.size = " << img.width() << " x " << img.height() << std::endl; + setPixmap(QPixmap::fromImage(img).scaled( QSize(height()*640/480,height()),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; } From c0614e70ac235724fe74085197e602f4978bfa47 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 9 Aug 2015 18:09:17 -0400 Subject: [PATCH 03/43] added preview option to show decoded stream in Audio config --- plugins/VOIP/gui/AudioInputConfig.cpp | 45 ++++++++++++----- plugins/VOIP/gui/AudioInputConfig.h | 2 + plugins/VOIP/gui/AudioInputConfig.ui | 10 ++++ plugins/VOIP/gui/QVideoDevice.cpp | 61 ++++++++++++++-------- plugins/VOIP/gui/QVideoDevice.h | 10 ++++ plugins/VOIP/gui/VideoProcessor.cpp | 73 ++++++++++++++++++++------- plugins/VOIP/gui/VideoProcessor.h | 53 +++++++++++++++---- plugins/VOIP/interface/rsVOIP.h | 4 +- 8 files changed, 195 insertions(+), 63 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index e23083200..d9931a2af 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -55,9 +55,9 @@ void AudioInputDialog::showEvent(QShowEvent *) { class voipGraphSource: public RSGraphSource { public: - voipGraphSource() {} + voipGraphSource() : video_input(NULL) {} - void setVideoInput(QVideoInputDevice *vid) { video_input = vid ; } + void setVideoInput(const QVideoInputDevice *vid) { video_input = vid ; } virtual QString displayName(int) const { return tr("Required bandwidth") ;} @@ -73,21 +73,14 @@ public: virtual void getValues(std::map& vals) const { - RsVOIPDataChunk chunk ; - uint32_t total_size = 0 ; vals.clear() ; - while(video_input && video_input->getNextEncodedPacket(chunk)) - { - total_size += chunk.size ; - chunk.clear() ; - } - - vals[std::string("bw")] = (float)total_size ; + if(video_input) + vals[std::string("bw")] = video_input->currentBandwidth() ; } private: - QVideoInputDevice *video_input ; + const QVideoInputDevice *video_input ; }; void voipGraph::setVoipSource(voipGraphSource *gs) @@ -127,12 +120,31 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) videoInput->setEchoVideoTarget(ui.videoDisplay) ; videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + videoDecoder = new JPEGVideoDecoder; + videoDecoder->setDisplayTarget(NULL) ; + graph_source = new voipGraphSource ; ui.voipBwGraph->setSource(graph_source); graph_source->setVideoInput(videoInput) ; graph_source->setCollectionTimeLimit(1000*300) ; graph_source->start() ; + + QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; +} + +void AudioInputConfig::togglePreview(bool b) +{ + if(b) + { + videoInput->setEchoVideoTarget(NULL) ; + videoDecoder->setDisplayTarget(ui.videoDisplay) ; + } + else + { + videoInput->setEchoVideoTarget(ui.videoDisplay) ; + videoDecoder->setDisplayTarget(NULL) ; + } } AudioInputConfig::~AudioInputConfig() @@ -352,6 +364,15 @@ void AudioInputConfig::on_Tick_timeout() { abSpeech->iValue = iroundf(inputAudioProcessor->dVoiceAcivityLevel * 32767.0f + 0.5f); abSpeech->update(); + + // also transmit encoded video + RsVOIPDataChunk chunk ; + + while(videoInput->getNextEncodedPacket(chunk)) + { + videoDecoder->receiveEncodedData(static_cast(chunk.data),chunk.size) ; + chunk.clear() ; + } } void AudioInputConfig::emptyBuffer() { diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index cc0fd15c3..faaadb63e 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -69,6 +69,7 @@ class AudioInputConfig : public ConfigPage //VideoDecoder *videoDecoder ; //VideoEncoder *videoEncoder ; QVideoInputDevice *videoInput ; + VideoDecoder *videoDecoder ; bool loaded; voipGraphSource *graph_source ; @@ -96,6 +97,7 @@ class AudioInputConfig : public ConfigPage private slots: void loadSettings(); void emptyBuffer(); + void togglePreview(bool) ; void on_qsTransmitHold_valueChanged(int v); void on_qsAmp_valueChanged(int v); diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 8037dea93..7b77fb3a9 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -391,6 +391,16 @@ + + + + <html><head/><body><p>Display encoded (and then decoded) frame, to check the codec's quality. If not selected, the image above only shows the frame that is grabbed from your camera.</p></body></html> + + + preview + + + diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 67c79ee5b..3a7ee029e 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -13,6 +13,9 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) _capture_device = NULL ; _video_encoder = NULL ; _echo_output_device = NULL ; + _estimated_bw = 0 ; + _total_encoded_size = 0 ; + _last_bw_estimate_TS = time(NULL) ; } void QVideoInputDevice::stop() @@ -54,36 +57,52 @@ void QVideoInputDevice::start() void QVideoInputDevice::grabFrame() { - IplImage *img=cvQueryFrame(_capture_device); + IplImage *img=cvQueryFrame(_capture_device); - if(img == NULL) - { - std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; - return ; - } - // get the image data + if(img == NULL) + { + std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; + return ; + } + // get the image data - if(img->nChannels != 3) - { - std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl; - return ; - } + if(img->nChannels != 3) + { + std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl; + return ; + } // convert to RGB and copy to new buffer, because cvQueryFrame tells us to not modify the buffer cv::Mat img_rgb; cv::cvtColor(cv::Mat(img), img_rgb, CV_BGR2RGB); - static const int _encoded_width = 128 ; - static const int _encoded_height = 128 ; + QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888); - QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888).scaled(QSize(_encoded_width,_encoded_height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + if(_video_encoder != NULL) + { + uint32_t encoded_size ; - if(_video_encoder != NULL) - { - _video_encoder->addImage(image) ; - emit networkPacketReady() ; - } - if(_echo_output_device != NULL) _echo_output_device->showFrame(image) ; + _video_encoder->addImage(image,0,encoded_size) ; + + std::cerr << "Encoded size = " << encoded_size << std::endl; + _total_encoded_size += encoded_size ; + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_TS) + { + _estimated_bw = uint32_t(0.75*_estimated_bw + 0.25 * (_total_encoded_size / (float)(now - _last_bw_estimate_TS))) ; + + _total_encoded_size = 0 ; + _last_bw_estimate_TS = now ; + + std::cerr << "new bw estimate: " << _estimated_bw << std::endl; + } + + emit networkPacketReady() ; + } + if(_echo_output_device != NULL) + _echo_output_device->showFrame(image) ; } bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index 7cb62ca0c..513932487 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -42,6 +42,12 @@ class QVideoInputDevice: public QObject // bool getNextEncodedPacket(RsVOIPDataChunk&) ; + // gets the estimated current bandwidth required to transmit the encoded data, in B/s + // + uint32_t currentBandwidth() const { return _estimated_bw ; } + + // control + void start() ; void stop() ; @@ -59,5 +65,9 @@ class QVideoInputDevice: public QObject QVideoOutputDevice *_echo_output_device ; std::list _out_queue ; + + uint32_t _estimated_bw ; + time_t _last_bw_estimate_TS; + uint32_t _total_encoded_size ; }; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 27343c412..c9add1b26 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -12,9 +12,15 @@ VideoDecoder::VideoDecoder() _output_device = NULL ; } -bool VideoEncoder::addImage(const QImage& img) +VideoEncoder::VideoEncoder() + :_frame_size(128,128) { - encodeData(img) ; +} + +bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) +{ + std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; + encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ; return true ; } @@ -32,6 +38,7 @@ bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk) void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) { + if(_output_device) _output_device->showFrame(decodeData(data,size)) ; } @@ -50,24 +57,54 @@ QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec) { - std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; + std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; } - -void JPEGVideoEncoder::encodeData(const QImage& image) +void VideoEncoder::setInternalFrameSize(QSize s) { - QByteArray qb ; - - QBuffer buffer(&qb) ; - buffer.open(QIODevice::WriteOnly) ; - image.save(&buffer,"JPEG") ; - - RsVOIPDataChunk voip_chunk ; - voip_chunk.data = malloc(qb.size()); - memcpy(voip_chunk.data,qb.data(),qb.size()) ; - voip_chunk.size = qb.size() ; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - _out_queue.push_back(voip_chunk) ; + _frame_size = s ; +} + +JPEGVideoEncoder::JPEGVideoEncoder() + : _ref_frame_max_distance(10),_ref_frame_count(10) +{ +} + +void JPEGVideoEncoder::encodeData(const QImage& image,uint32_t /* size_hint */,uint32_t& encoded_size) +{ + // check if we make a diff image, or if we use the full frame. + + QImage encoded_frame ; + + if(_ref_frame_count++ < _ref_frame_max_distance && image.size() == _reference_frame.size()) + { + // compute difference with reference frame. + encoded_frame = image ; + + for(uint32_t i=0;i _out_queue ; + + QSize _frame_size ; }; // Now derive various image encoding/decoding algorithms. @@ -75,16 +77,45 @@ class VideoEncoder class JPEGVideoDecoder: public VideoDecoder { - protected: - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; +protected: + virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; + +private: + QImage _last_reference_frame ; }; class JPEGVideoEncoder: public VideoEncoder { - public: - JPEGVideoEncoder() {} +public: + JPEGVideoEncoder() ; - protected: - virtual void encodeData(const QImage& Image) ; +protected: + virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + +private: + QImage _reference_frame ; + uint32_t _ref_frame_max_distance ; // max distance between two reference frames. + uint32_t _ref_frame_count ; }; +class DifferentialWaveletEncoder: public VideoEncoder +{ +public: + DifferentialWaveletEncoder() {} + +protected: + virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + +}; + +class DifferentialWaveletDecoder: public VideoDecoder +{ +public: + DifferentialWaveletDecoder() {} + +protected: + virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; + +private: + QImage _last_reference_frame ; +}; diff --git a/plugins/VOIP/interface/rsVOIP.h b/plugins/VOIP/interface/rsVOIP.h index 7269610b4..25fd0cfbd 100644 --- a/plugins/VOIP/interface/rsVOIP.h +++ b/plugins/VOIP/interface/rsVOIP.h @@ -51,7 +51,9 @@ class RsVOIPPongResult struct RsVOIPDataChunk { - typedef enum { RS_VOIP_DATA_TYPE_AUDIO, RS_VOIP_DATA_TYPE_VIDEO } RsVOIPDataType ; + typedef enum { RS_VOIP_DATA_TYPE_UNKNOWN = 0x00, + RS_VOIP_DATA_TYPE_AUDIO = 0x01, + RS_VOIP_DATA_TYPE_VIDEO = 0x02 } RsVOIPDataType ; void *data ; // create/delete using malloc/free. uint32_t size ; From 3bda2c2660131e5694d8375527c2262c52db53eb Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 10 Aug 2015 22:13:50 -0400 Subject: [PATCH 04/43] added differential frames encoding/decoding, and bandwidth items --- plugins/VOIP/gui/AudioInputConfig.cpp | 57 +++++++++--------- plugins/VOIP/gui/VideoProcessor.cpp | 83 +++++++++++++++++++++----- plugins/VOIP/gui/VideoProcessor.h | 14 +++-- plugins/VOIP/services/rsVOIPItems.cc | 85 +++++++++++++++++++++++++++ plugins/VOIP/services/rsVOIPItems.h | 29 +++++++-- 5 files changed, 213 insertions(+), 55 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index d9931a2af..29a27026a 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -29,17 +29,14 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#include -//#include "AudioInput.h" -//#include "AudioOutput.h" #include "AudioStats.h" #include "AudioInputConfig.h" -//#include "Global.h" -//#include "NetworkConfig.h" #include "audiodevicehelper.h" #include "AudioWizard.h" #include "gui/common/RSGraphWidget.h" +#include "util/RsProtectedTimer.h" + #include #define iroundf(x) ( static_cast(x) ) @@ -104,33 +101,33 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) { std::cerr << "Creating audioInputConfig object" << std::endl; - /* Invoke the Qt Designer generated object setup routine */ - ui.setupUi(this); + /* Invoke the Qt Designer generated object setup routine */ + ui.setupUi(this); - loaded = false; + loaded = false; - inputAudioProcessor = NULL; - inputAudioDevice = NULL; - abSpeech = NULL; - qtTick = NULL; + inputAudioProcessor = NULL; + inputAudioDevice = NULL; + abSpeech = NULL; + qtTick = NULL; - // Create the video pipeline. - // - videoInput = new QVideoInputDevice(this) ; - videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; - - videoDecoder = new JPEGVideoDecoder; - videoDecoder->setDisplayTarget(NULL) ; - - graph_source = new voipGraphSource ; - ui.voipBwGraph->setSource(graph_source); - - graph_source->setVideoInput(videoInput) ; - graph_source->setCollectionTimeLimit(1000*300) ; - graph_source->start() ; - - QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; + // Create the video pipeline. + // + videoInput = new QVideoInputDevice(this) ; + videoInput->setEchoVideoTarget(ui.videoDisplay) ; + videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + + videoDecoder = new JPEGVideoDecoder; + videoDecoder->setDisplayTarget(NULL) ; + + graph_source = new voipGraphSource ; + ui.voipBwGraph->setSource(graph_source); + + graph_source->setVideoInput(videoInput) ; + graph_source->setCollectionTimeLimit(1000*300) ; + graph_source->start() ; + + QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; } void AudioInputConfig::togglePreview(bool b) @@ -178,7 +175,7 @@ void AudioInputConfig::load() //connect( ui.allowIpDeterminationCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleIpDetermination(bool) ) ); //connect( ui.allowTunnelConnectionCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleTunnelConnection(bool) ) ); - qtTick = new QTimer(this); + qtTick = new RsProtectedTimer(this); connect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) ); qtTick->start(20); /*if (AudioInputRegistrar::qmNew) { diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index c9add1b26..b19ee03d0 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -13,14 +13,15 @@ VideoDecoder::VideoDecoder() } VideoEncoder::VideoEncoder() - :_frame_size(128,128) + :_frame_size(256,256) { } bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) { - std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; +// std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ; + //encodeData(img,size_hint,encoded_size) ; return true ; } @@ -44,15 +45,51 @@ void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint32_t size) { - QByteArray qb((char*)encoded_image_data,size) ; - QImage image ; - if(image.loadFromData(qb,"JPEG")) - return image ; - else - { - std::cerr << "image.loadFromData(): returned an error.: " << std::endl; - return QImage() ; - } + static const int HEADER_SIZE = 4 ; + + // read frame type. Use first 4 bytes to give info about content. + + if(size < HEADER_SIZE) + { + std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << size << std::endl; + return QImage() ; + } + + uint32_t flags = encoded_image_data[0] + (encoded_image_data[1] << 8) ; + + // un-compress image data + + QByteArray qb((char*)&encoded_image_data[HEADER_SIZE],(int)size - HEADER_SIZE) ; + QImage image ; + if(!image.loadFromData(qb,"JPEG")) + { + std::cerr << "image.loadFromData(): returned an error.: " << std::endl; + return QImage() ; + } + + // now see if the frame is a differential frame, or just a reference frame. + + if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) + { + if(_reference_frame.size() != image.size()) + { + std::cerr << "Bad reference frame!" << std::endl; + return image ; + } + + QImage res = _reference_frame ; + + for(uint32_t i=0;i> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = 0 ; + ((unsigned char *)voip_chunk.data)[3] = 0 ; + + memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; + voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; _out_queue.push_back(voip_chunk) ; diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index d78b1ad9d..42124b4f4 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -80,8 +80,11 @@ class JPEGVideoDecoder: public VideoDecoder protected: virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; + static const uint32_t HEADER_SIZE = 0x04 ; + + static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; private: - QImage _last_reference_frame ; + QImage _reference_frame ; }; class JPEGVideoEncoder: public VideoEncoder @@ -92,10 +95,13 @@ public: protected: virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + static const uint32_t HEADER_SIZE = 0x04 ; + + static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; private: - QImage _reference_frame ; - uint32_t _ref_frame_max_distance ; // max distance between two reference frames. - uint32_t _ref_frame_count ; + QImage _reference_frame ; + uint32_t _ref_frame_max_distance ; // max distance between two reference frames. + uint32_t _ref_frame_count ; }; class DifferentialWaveletEncoder: public VideoEncoder diff --git a/plugins/VOIP/services/rsVOIPItems.cc b/plugins/VOIP/services/rsVOIPItems.cc index eb8842468..a75fa4c16 100644 --- a/plugins/VOIP/services/rsVOIPItems.cc +++ b/plugins/VOIP/services/rsVOIPItems.cc @@ -76,6 +76,19 @@ std::ostream& RsVOIPProtocolItem::print(std::ostream &out, uint16_t indent) printRsItemEnd(out, "RsVOIPProtocolItem", indent); return out; } +std::ostream& RsVOIPBandwidthItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsVOIPBandwidthItem", indent); + uint16_t int_Indent = indent + 2; + printIndent(out, int_Indent); + out << "flags: " << std::hex << flags << std::dec << std::endl; + + printIndent(out, int_Indent); + out << "speed: " << bytes_per_sec << std::endl; + + printRsItemEnd(out, "RsVOIPBandwidthItem", indent); + return out; +} std::ostream& RsVOIPDataItem::print(std::ostream &out, uint16_t indent) { printRsItemBase(out, "RsVOIPDataItem", indent); @@ -100,6 +113,14 @@ uint32_t RsVOIPDataItem::serial_size() const return s; } +uint32_t RsVOIPBandwidthItem::serial_size() const +{ + uint32_t s = 8; /* header */ + s += 4; /* flags */ + s += 4; /* bandwidth */ + + return s; +} uint32_t RsVOIPProtocolItem::serial_size() const { uint32_t s = 8; /* header */ @@ -150,6 +171,40 @@ bool RsVOIPProtocolItem::serialise(void *data, uint32_t& pktsize) return ok; } +bool RsVOIPBandwidthItem::serialise(void *data, uint32_t& pktsize) +{ + uint32_t tlvsize = serial_size() ; + uint32_t offset = 0; + + if (pktsize < tlvsize) + return false; /* not enough space */ + + pktsize = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, PacketId(), tlvsize); + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Header: " << ok << std::endl; + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size: " << tlvsize << std::endl; +#endif + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= setRawUInt32(data, tlvsize, &offset, flags); + ok &= setRawUInt32(data, tlvsize, &offset, bytes_per_sec); + + if (offset != tlvsize) + { + ok = false; + std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size Error! " << std::endl; + } + + return ok; +} /* serialise the data to the buffer */ bool RsVOIPDataItem::serialise(void *data, uint32_t& pktsize) { @@ -254,6 +309,36 @@ RsVOIPProtocolItem::RsVOIPProtocolItem(void *data, uint32_t pktsize) if (!ok) throw std::runtime_error("Deserialisation error!") ; } +RsVOIPBandwidthItem::RsVOIPBandwidthItem(void *data, uint32_t pktsize) + : RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH) +{ + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + uint32_t offset = 0; + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_VOIP_PLUGIN != getRsItemService(rstype)) || (RS_PKT_SUBTYPE_VOIP_BANDWIDTH != getRsItemSubType(rstype))) + throw std::runtime_error("Wrong packet type!") ; + + if (pktsize < rssize) /* check size */ + throw std::runtime_error("Not enough size!") ; + + bool ok = true; + + /* skip the header */ + offset += 8; + + /* get mandatory parts first */ + ok &= getRawUInt32(data, rssize, &offset, &flags); + ok &= getRawUInt32(data, rssize, &offset, &bytes_per_sec); + + if (offset != rssize) + throw std::runtime_error("Deserialisation error!") ; + + if (!ok) + throw std::runtime_error("Deserialisation error!") ; +} RsVOIPPingItem::RsVOIPPingItem(void *data, uint32_t pktsize) : RsVOIPItem(RS_PKT_SUBTYPE_VOIP_PING) { diff --git a/plugins/VOIP/services/rsVOIPItems.h b/plugins/VOIP/services/rsVOIPItems.h index fa440ef19..7658ea4cc 100644 --- a/plugins/VOIP/services/rsVOIPItems.h +++ b/plugins/VOIP/services/rsVOIPItems.h @@ -55,11 +55,12 @@ const uint16_t RS_SERVICE_TYPE_VOIP_PLUGIN = 0xa021; -const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; -const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; -const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ; - // 0x04 is unused because of a change in the protocol -const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x05 ; +const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01; +const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02; +const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ; + // 0x04,0x05 is unused because of a change in the protocol +const uint8_t RS_PKT_SUBTYPE_VOIP_BANDWIDTH = 0x06 ; +const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x07 ; const uint8_t QOS_PRIORITY_RS_VOIP = 9 ; @@ -117,9 +118,27 @@ class RsVOIPDataItem: public RsVOIPItem uint32_t flags ; uint32_t data_size ; + void *voip_data ; }; +class RsVOIPBandwidthItem: public RsVOIPItem +{ + public: + RsVOIPBandwidthItem() :RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH) {} + RsVOIPBandwidthItem(void *data,uint32_t size) ; // de-serialization + + virtual bool serialise(void *data,uint32_t& size) ; + virtual uint32_t serial_size() const ; + + virtual ~RsVOIPBandwidthItem() {} + virtual std::ostream& print(std::ostream &out, uint16_t indent = 0); + + uint32_t flags ; // is that incoming or expected bandwidth? + uint32_t bytes_per_sec ; // bandwidth in bytes per sec. +}; + + class RsVOIPProtocolItem: public RsVOIPItem { public: From cd97fd9682aebb365bfd808693f8a23a12923739 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 14 Aug 2015 16:44:20 -0400 Subject: [PATCH 05/43] refactored video processing classes, to allow multiple codecs. Not working yet. --- plugins/VOIP/gui/AudioInputConfig.cpp | 50 +++--- plugins/VOIP/gui/AudioInputConfig.h | 2 +- plugins/VOIP/gui/QVideoDevice.cpp | 10 +- plugins/VOIP/gui/QVideoDevice.h | 5 +- plugins/VOIP/gui/VOIPChatWidgetHolder.cpp | 80 +++++---- plugins/VOIP/gui/VOIPChatWidgetHolder.h | 6 +- plugins/VOIP/gui/VideoProcessor.cpp | 200 ++++++++++++++-------- plugins/VOIP/gui/VideoProcessor.h | 168 +++++++++--------- 8 files changed, 284 insertions(+), 237 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 29a27026a..5f6aacfb8 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -34,6 +34,7 @@ #include "AudioInputConfig.h" #include "audiodevicehelper.h" #include "AudioWizard.h" +#include "gui/VideoProcessor.h" #include "gui/common/RSGraphWidget.h" #include "util/RsProtectedTimer.h" @@ -99,35 +100,36 @@ voipGraph::voipGraph(QWidget *parent) AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) : ConfigPage(parent, flags) { - std::cerr << "Creating audioInputConfig object" << std::endl; + std::cerr << "Creating audioInputConfig object" << std::endl; - /* Invoke the Qt Designer generated object setup routine */ - ui.setupUi(this); + /* Invoke the Qt Designer generated object setup routine */ + ui.setupUi(this); - loaded = false; + loaded = false; - inputAudioProcessor = NULL; - inputAudioDevice = NULL; - abSpeech = NULL; - qtTick = NULL; + inputAudioProcessor = NULL; + inputAudioDevice = NULL; + abSpeech = NULL; + qtTick = NULL; - // Create the video pipeline. - // - videoInput = new QVideoInputDevice(this) ; - videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoInput->setVideoEncoder(new JPEGVideoEncoder()) ; + // Create the video pipeline. + // + videoInput = new QVideoInputDevice(this) ; + videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoDecoder = new JPEGVideoDecoder; - videoDecoder->setDisplayTarget(NULL) ; + videoProcessor = new VideoProcessor() ; + videoProcessor->setDisplayTarget(NULL) ; - graph_source = new voipGraphSource ; - ui.voipBwGraph->setSource(graph_source); + videoInput->setVideoProcessor(videoProcessor) ; - graph_source->setVideoInput(videoInput) ; - graph_source->setCollectionTimeLimit(1000*300) ; - graph_source->start() ; + graph_source = new voipGraphSource ; + ui.voipBwGraph->setSource(graph_source); - QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; + graph_source->setVideoInput(videoInput) ; + graph_source->setCollectionTimeLimit(1000*300) ; + graph_source->start() ; + + QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; } void AudioInputConfig::togglePreview(bool b) @@ -135,12 +137,12 @@ void AudioInputConfig::togglePreview(bool b) if(b) { videoInput->setEchoVideoTarget(NULL) ; - videoDecoder->setDisplayTarget(ui.videoDisplay) ; + videoProcessor->setDisplayTarget(ui.videoDisplay) ; } else { videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoDecoder->setDisplayTarget(NULL) ; + videoProcessor->setDisplayTarget(NULL) ; } } @@ -367,7 +369,7 @@ void AudioInputConfig::on_Tick_timeout() { while(videoInput->getNextEncodedPacket(chunk)) { - videoDecoder->receiveEncodedData(static_cast(chunk.data),chunk.size) ; + videoProcessor->receiveEncodedData(chunk) ; chunk.clear() ; } } diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index faaadb63e..e838744b4 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -69,7 +69,7 @@ class AudioInputConfig : public ConfigPage //VideoDecoder *videoDecoder ; //VideoEncoder *videoEncoder ; QVideoInputDevice *videoInput ; - VideoDecoder *videoDecoder ; + VideoProcessor *videoProcessor ; bool loaded; voipGraphSource *graph_source ; diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 3a7ee029e..e9ace94f9 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -11,7 +11,7 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) { _timer = NULL ; _capture_device = NULL ; - _video_encoder = NULL ; + _video_processor = NULL ; _echo_output_device = NULL ; _estimated_bw = 0 ; _total_encoded_size = 0 ; @@ -78,11 +78,11 @@ void QVideoInputDevice::grabFrame() QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888); - if(_video_encoder != NULL) + if(_video_processor != NULL) { uint32_t encoded_size ; - _video_encoder->addImage(image,0,encoded_size) ; + _video_processor->processImage(image,0,encoded_size) ; std::cerr << "Encoded size = " << encoded_size << std::endl; _total_encoded_size += encoded_size ; @@ -107,8 +107,8 @@ void QVideoInputDevice::grabFrame() bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { - if(_video_encoder) - return _video_encoder->nextPacket(chunk) ; + if(_video_processor) + return _video_processor->nextEncodedPacket(chunk) ; else return false ; } diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index 513932487..d12fe5f38 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -2,6 +2,7 @@ #include #include "interface/rsVOIP.h" +#include "gui/VideoProcessor.h" class VideoEncoder ; class CvCapture ; @@ -31,7 +32,7 @@ class QVideoInputDevice: public QObject // Captured images are sent to this encoder. Can be NULL. // - void setVideoEncoder(VideoEncoder *venc) { _video_encoder = venc ; } + void setVideoProcessor(VideoProcessor *venc) { _video_processor = venc ; } // All images received will be echoed to this target. We could use signal/slots, but it's // probably faster this way. Can be NULL. @@ -58,7 +59,7 @@ class QVideoInputDevice: public QObject void networkPacketReady() ; private: - VideoEncoder *_video_encoder ; + VideoProcessor *_video_processor ; QTimer *_timer ; CvCapture *_capture_device ; diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp index 1d94960d7..4d1f8ff49 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp @@ -123,8 +123,7 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n inputAudioDevice = NULL ; inputVideoDevice = new QVideoInputDevice(mChatWidget) ; // not started yet ;-) - inputVideoProcessor = new JPEGVideoEncoder ; - outputVideoProcessor = new JPEGVideoDecoder ; + videoProcessor = new VideoProcessor ; // Make a widget with two video devices, one for echo, and one for the talking peer. videoWidget = new QWidget(mChatWidget) ; @@ -144,8 +143,8 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n mChatWidget->addChatHorizontalWidget(videoWidget) ; inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ; - inputVideoDevice->setVideoEncoder(inputVideoProcessor) ; - outputVideoProcessor->setDisplayTarget(outputVideoDevice) ; + inputVideoDevice->setVideoProcessor(videoProcessor) ; + videoProcessor->setDisplayTarget(outputVideoDevice) ; } VOIPChatWidgetHolder::~VOIPChatWidgetHolder() @@ -154,8 +153,7 @@ VOIPChatWidgetHolder::~VOIPChatWidgetHolder() inputAudioDevice->stop() ; delete inputVideoDevice ; - delete inputVideoProcessor ; - delete outputVideoProcessor ; + delete videoProcessor ; button_map::iterator it = buttonMapTakeVideo.begin(); while (it != buttonMapTakeVideo.end()) { @@ -287,42 +285,50 @@ void VOIPChatWidgetHolder::toggleVideoCapture() void VOIPChatWidgetHolder::addVideoData(const RsPeerId &peer_id, QByteArray* array) { - if (!videoCaptureToggleButton->isChecked()) { - if (mChatWidget) { - QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str()); - if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId - button_map::iterator it = buttonMapTakeVideo.find(buttonName); - if (it == buttonMapTakeVideo.end()){ - mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime() - , tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM); - RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call")); - button->setToolTip(tr("Activate camera")); - button->setStyleSheet(QString("border: 1px solid #199909;") - .append("font-size: 12pt; color: white;") - .append("min-width: 128px; min-height: 24px;") - .append("border-radius: 6px;") - .append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, " - "stop: 0 #22c70d, stop: 1 #116a06);") + if (!videoCaptureToggleButton->isChecked()) + { + if (mChatWidget) { + QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str()); + if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId + button_map::iterator it = buttonMapTakeVideo.find(buttonName); + if (it == buttonMapTakeVideo.end()){ + mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime() + , tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM); + RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call")); + button->setToolTip(tr("Activate camera")); + button->setStyleSheet(QString("border: 1px solid #199909;") + .append("font-size: 12pt; color: white;") + .append("min-width: 128px; min-height: 24px;") + .append("border-radius: 6px;") + .append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, " + "stop: 0 #22c70d, stop: 1 #116a06);") - ); + ); - button->updateImage(); + button->updateImage(); - connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture())); - connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter())); - connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave())); + connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture())); + connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter())); + connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave())); - buttonMapTakeVideo.insert(buttonName, button); - } - } + buttonMapTakeVideo.insert(buttonName, button); + } + } - //TODO make a sound for the incoming call -// soundManager->play(VOIP_SOUND_INCOMING_CALL); - if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id); + //TODO make a sound for the incoming call + // soundManager->play(VOIP_SOUND_INCOMING_CALL); + if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id); - } else { - outputVideoProcessor->receiveEncodedData((unsigned char *)array->data(),array->size()) ; - } + } + else + { + RsVOIPDataChunk chunk ; + chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + chunk.size = array->size() ; + chunk.data = array->data() ; + + videoProcessor->receiveEncodedData(chunk) ; + } } void VOIPChatWidgetHolder::botMouseEnter() @@ -359,7 +365,7 @@ void VOIPChatWidgetHolder::botMouseLeave() void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec) { - inputVideoProcessor->setMaximumFrameRate(bytes_per_sec) ; + videoProcessor->setMaximumFrameRate(bytes_per_sec) ; } void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array) diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.h b/plugins/VOIP/gui/VOIPChatWidgetHolder.h index 791a2dcf8..350f49f2c 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.h +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.h @@ -34,8 +34,7 @@ class QAudioInput; class QAudioOutput; class QVideoInputDevice ; class QVideoOutputDevice ; -class VideoEncoder ; -class VideoDecoder ; +class VideoProcessor ; #define VOIP_SOUND_INCOMING_CALL "VOIP_incoming_call" @@ -82,8 +81,7 @@ protected: QWidget *videoWidget ; // pointer to call show/hide - VideoEncoder *inputVideoProcessor; - VideoDecoder *outputVideoProcessor; + VideoProcessor *videoProcessor; // Additional buttons to the chat bar QToolButton *audioListenToggleButton ; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index b19ee03d0..b3d0a7c27 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -7,127 +8,182 @@ #include "VideoProcessor.h" #include "QVideoDevice.h" -VideoDecoder::VideoDecoder() +VideoProcessor::VideoProcessor() + :_encoded_frame_size(256,256) { - _output_device = NULL ; + _decoded_output_device = NULL ; } -VideoEncoder::VideoEncoder() - :_frame_size(256,256) +bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) { + VideoCodec *codec ; + + switch(_encoding_current_codec) + { + case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; + break ; + case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; + break ; + default: + codec = NULL ; + } + + // std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; + + void *data = NULL; + encoded_size = 0 ; + + if(codec) + { + RsVOIPDataChunk chunk ; + + codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) ; + + if(chunk.size == 0) // the codec might be buffering the frame for compression reasons + return true ; + + _encoded_out_queue.push_back(chunk) ; + + return true ; + } + else + return false ; } -bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) +bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) { -// std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; - encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ; - //encodeData(img,size_hint,encoded_size) ; - - return true ; -} - -bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk) -{ - if(_out_queue.empty()) + if(_encoded_out_queue.empty()) return false ; - chunk = _out_queue.front() ; - _out_queue.pop_front() ; + chunk = _encoded_out_queue.front() ; + _encoded_out_queue.pop_front() ; return true ; } -void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) +void VideoProcessor::setInternalFrameSize(QSize s) { - if(_output_device) - _output_device->showFrame(decodeData(data,size)) ; + _encoded_frame_size = s ; } -QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint32_t size) +void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) { static const int HEADER_SIZE = 4 ; // read frame type. Use first 4 bytes to give info about content. + // + // Byte Meaning Values + // 00 Codec CODEC_ID_JPEG_VIDEO Basic Jpeg codec + // CODEC_ID_DDWT_VIDEO Differential wavelet compression + // + // 01 Unused Might be useful later + // + // 0203 Flags Codec specific flags. + // - if(size < HEADER_SIZE) + if(chunk.size < HEADER_SIZE) { - std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << size << std::endl; - return QImage() ; + std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << chunk.size << std::endl; + return ; } - uint32_t flags = encoded_image_data[0] + (encoded_image_data[1] << 8) ; + uint32_t codid = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; + uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; - // un-compress image data + VideoCodec *codec ; - QByteArray qb((char*)&encoded_image_data[HEADER_SIZE],(int)size - HEADER_SIZE) ; - QImage image ; - if(!image.loadFromData(qb,"JPEG")) + switch(codid) { - std::cerr << "image.loadFromData(): returned an error.: " << std::endl; - return QImage() ; + case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; + break ; + case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; + break ; + default: + codec = NULL ; } + QImage img ; - // now see if the frame is a differential frame, or just a reference frame. + if(codec != NULL) + codec->decodeData(chunk,img) ; - if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) - { - if(_reference_frame.size() != image.size()) - { - std::cerr << "Bad reference frame!" << std::endl; - return image ; - } - - QImage res = _reference_frame ; - - for(uint32_t i=0;ishowFrame(img) ; } -void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec) +void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) { std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; } -void VideoEncoder::setInternalFrameSize(QSize s) -{ - _frame_size = s ; -} +////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// +////////////////////////////////////////////////////////////////////////////////////////////////////////////// -JPEGVideoEncoder::JPEGVideoEncoder() - : _ref_frame_max_distance(50),_ref_frame_count(50) +JPEGVideo::JPEGVideo() + : _encoded_ref_frame_max_distance(50),_encoded_ref_frame_count(50) { } -void JPEGVideoEncoder::encodeData(const QImage& image,uint32_t /* size_hint */,uint32_t& encoded_size) +bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) +{ + // now see if the frame is a differential frame, or just a reference frame. + + uint16_t codec = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; + uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; + + assert(codec == VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO) ; + + // un-compress image data + + QByteArray qb((char*)&((uint8_t*)chunk.data)[HEADER_SIZE],(int)chunk.size - HEADER_SIZE) ; + + if(!image.loadFromData(qb,"JPEG")) + { + std::cerr << "image.loadFromData(): returned an error.: " << std::endl; + return false ; + } + + + if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) + { + if(_decoded_reference_frame.size() != image.size()) + { + std::cerr << "Bad reference frame!" << std::endl; + return false ; + } + + QImage res = _decoded_reference_frame ; + + for(uint32_t i=0;i> 8) & 0xff ; - ((unsigned char *)voip_chunk.data)[2] = 0 ; - ((unsigned char *)voip_chunk.data)[3] = 0 ; + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; + voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - _out_queue.push_back(voip_chunk) ; - - encoded_size = voip_chunk.size ; + return true ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 42124b4f4..d103207af 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -6,58 +6,91 @@ class QVideoOutputDevice ; +class VideoCodec +{ +public: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) = 0; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) = 0; +}; + +// Now derive various image encoding/decoding algorithms. +// + +class JPEGVideo: public VideoCodec +{ +public: + JPEGVideo() ; + +protected: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; + + static const uint32_t HEADER_SIZE = 0x04 ; + static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; +private: + QImage _decoded_reference_frame ; + QImage _encoded_reference_frame ; + + uint32_t _encoded_ref_frame_max_distance ; // max distance between two reference frames. + uint32_t _encoded_ref_frame_count ; +}; + +class DifferentialWaveletVideo: public VideoCodec +{ +public: + DifferentialWaveletVideo() {} + +protected: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) { return true ; } + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) { return true ; } + +private: + QImage _last_reference_frame ; +}; + // This class decodes video from a stream. It keeps a queue of // decoded frame that needs to be retrieved using the getNextImage() method. // -class VideoDecoder +class VideoProcessor { public: - VideoDecoder() ; - virtual ~VideoDecoder() {} + VideoProcessor() ; + virtual ~VideoProcessor() {} + enum CodecId { + VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000, + VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO = 0x0001, + VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002 + }; + +// ===================================================================================== +// =------------------------------------ DECODING -------------------------------------= +// ===================================================================================== + // Gets the next image to be displayed. Once returned, the image should // be cleared from the incoming queue. // - void setDisplayTarget(QVideoOutputDevice *odev) { _output_device = odev ; } - - virtual void receiveEncodedData(const unsigned char *data,uint32_t size) ; + void setDisplayTarget(QVideoOutputDevice *odev) { _decoded_output_device = odev ; } + virtual void receiveEncodedData(const RsVOIPDataChunk& chunk) ; // returns the current (measured) frame rate in bytes per second. // - uint32_t currentFrameRate() const; + uint32_t currentDecodingFrameRate() const; private: - QVideoOutputDevice *_output_device ; + QVideoOutputDevice *_decoded_output_device ; + std::list _decoded_image_queue ; - std::list _image_queue ; - - // Incoming data is processed by a video codec and converted into images. - // - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) = 0 ; - -// // This buffer accumulated incoming encoded data, until a full packet is obtained, -// // since the stream might not send images at once. When incoming images are decoded, the -// // data is removed from the buffer. -// // -// unsigned char *buffer ; -// uint32_t buffer_size ; -}; - -// This class encodes video using a video codec (possibly homemade, or based on existing codecs) -// and produces a data stream that is sent to the network transfer service (e.g. p3VOIP). -// -class VideoEncoder -{ +// ===================================================================================== +// =------------------------------------ ENCODING -------------------------------------= +// ===================================================================================== + public: - VideoEncoder() ; - virtual ~VideoEncoder() {} - // Takes the next image to be encoded. // - bool addImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; - - bool packetReady() const { return !_out_queue.empty() ; } - bool nextPacket(RsVOIPDataChunk& ) ; + bool processImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; } + bool nextEncodedPacket(RsVOIPDataChunk& ) ; // Used to tweak the compression ratio so that the video can stream ok. // @@ -65,63 +98,16 @@ class VideoEncoder void setInternalFrameSize(QSize) ; protected: - virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) =0; - - std::list _out_queue ; + std::list _encoded_out_queue ; + QSize _encoded_frame_size ; + +// ===================================================================================== +// =------------------------------------- Codecs --------------------------------------= +// ===================================================================================== - QSize _frame_size ; + JPEGVideo _jpeg_video_codec ; + DifferentialWaveletVideo _ddwt_video_codec ; + + uint16_t _encoding_current_codec ; }; -// Now derive various image encoding/decoding algorithms. -// - -class JPEGVideoDecoder: public VideoDecoder -{ -protected: - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; - - static const uint32_t HEADER_SIZE = 0x04 ; - - static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; -private: - QImage _reference_frame ; -}; - -class JPEGVideoEncoder: public VideoEncoder -{ -public: - JPEGVideoEncoder() ; - -protected: - virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; - - static const uint32_t HEADER_SIZE = 0x04 ; - - static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; -private: - QImage _reference_frame ; - uint32_t _ref_frame_max_distance ; // max distance between two reference frames. - uint32_t _ref_frame_count ; -}; - -class DifferentialWaveletEncoder: public VideoEncoder -{ -public: - DifferentialWaveletEncoder() {} - -protected: - virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; - -}; - -class DifferentialWaveletDecoder: public VideoDecoder -{ -public: - DifferentialWaveletDecoder() {} - -protected: - virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ; - -private: - QImage _last_reference_frame ; -}; From 5aac92fc97bf976a954d5381f4bce168b2015ff7 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 14 Aug 2015 20:15:44 -0400 Subject: [PATCH 06/43] fixed bug in differential frame encoding --- plugins/VOIP/gui/VideoProcessor.cpp | 35 ++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index b3d0a7c27..7a43058df 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -12,6 +12,7 @@ VideoProcessor::VideoProcessor() :_encoded_frame_size(256,256) { _decoded_output_device = NULL ; + _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; } bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) @@ -47,7 +48,10 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& return true ; } else + { + std::cerr << "No codec for codec ID = " << _encoding_current_codec << ". Please call VideoProcessor::setCurrentCodec()" << std::endl; return false ; + } } bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) @@ -120,7 +124,7 @@ void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) ////////////////////////////////////////////////////////////////////////////////////////////////////////////// JPEGVideo::JPEGVideo() - : _encoded_ref_frame_max_distance(50),_encoded_ref_frame_count(50) + : _encoded_ref_frame_max_distance(10),_encoded_ref_frame_count(10) { } @@ -146,19 +150,25 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) { - if(_decoded_reference_frame.size() != image.size()) - { - std::cerr << "Bad reference frame!" << std::endl; - return false ; - } - + if(_decoded_reference_frame.size() != image.size()) + { + std::cerr << "Bad reference frame!" << std::endl; + return false ; + } + QImage res = _decoded_reference_frame ; for(uint32_t i=0;i Date: Fri, 14 Aug 2015 22:44:39 -0400 Subject: [PATCH 07/43] restored original params for JPEG codec --- plugins/VOIP/gui/VideoProcessor.cpp | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 7a43058df..4e1979a5d 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -9,7 +9,7 @@ #include "QVideoDevice.h" VideoProcessor::VideoProcessor() - :_encoded_frame_size(256,256) + :_encoded_frame_size(128,128) { _decoded_output_device = NULL ; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; @@ -40,6 +40,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) ; + encoded_size = chunk.size ; + if(chunk.size == 0) // the codec might be buffering the frame for compression reasons return true ; @@ -146,7 +148,6 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << "image.loadFromData(): returned an error.: " << std::endl; return false ; } - if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) { @@ -187,6 +188,9 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa for(uint32_t i=0;i> 8) & 0xff ; ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; - + memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; - + voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; From d28c1898fd74d09c455a34f1754bc0fb9916e207 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 16 Aug 2015 22:59:49 -0400 Subject: [PATCH 08/43] first workign implementation of wavelet-based codec --- plugins/VOIP/gui/DaubechyWavelets.h | 267 ++++++++++++++++++++++++++++ plugins/VOIP/gui/VideoProcessor.cpp | 240 ++++++++++++++++++++++++- plugins/VOIP/gui/VideoProcessor.h | 25 ++- 3 files changed, 519 insertions(+), 13 deletions(-) create mode 100644 plugins/VOIP/gui/DaubechyWavelets.h diff --git a/plugins/VOIP/gui/DaubechyWavelets.h b/plugins/VOIP/gui/DaubechyWavelets.h new file mode 100644 index 000000000..c69f025b0 --- /dev/null +++ b/plugins/VOIP/gui/DaubechyWavelets.h @@ -0,0 +1,267 @@ +#pragma once + +#include +#include +#include +#include +#include + +#ifdef USE_SSE_INSTRUCTIONS +#include +#endif + +template class DaubechyWavelets +{ + public: + typedef enum { DWT_DAUB02=2, DWT_DAUB04=4, DWT_DAUB12=12, DWT_DAUB20=20 } WaveletType ; + typedef enum { DWT_FORWARD=1, DWT_BACKWARD=0 } TransformType ; + + static void DWT2D(FLOAT *data,unsigned long int W,unsigned long int H,WaveletType type,TransformType tr) + { + unsigned long int nn[2] = {W,H} ; + wtn(&data[-1], &nn[-1],2, tr, waveletFilter(type), pwt) ; + } + static void DWT1D(FLOAT *data,unsigned long int W,WaveletType type,TransformType tr) + { + unsigned long int nn[1] = {W} ; + wtn(&data[-1], &nn[-1],1, tr, waveletFilter(type), pwt) ; + } + + + private: + class wavefilt + { + public: + wavefilt(int n) + { + int k; + FLOAT sig = -1.0; + static const FLOAT c2[5]={ 0.0, sqrt(2.0)/2.0, sqrt(2.0)/2.0, 0.0, 0.0 }; + + static const FLOAT c4[5]={ 0.0, 0.4829629131445341, 0.8365163037378079, 0.2241438680420134,-0.1294095225512604 }; + + static const FLOAT c12[13]={0.0,0.111540743350, 0.494623890398, 0.751133908021, + 0.315250351709,-0.226264693965,-0.129766867567, + 0.097501605587, 0.027522865530,-0.031582039318, + 0.000553842201, 0.004777257511,-0.001077301085}; + + static const FLOAT c20[21]={0.0,0.026670057901, 0.188176800078, 0.527201188932, + 0.688459039454, 0.281172343661,-0.249846424327, + -0.195946274377, 0.127369340336, 0.093057364604, + -0.071394147166,-0.029457536822, 0.033212674059, + 0.003606553567,-0.010733175483, 0.001395351747, + 0.001992405295,-0.000685856695,-0.000116466855, + 0.000093588670,-0.000013264203 }; + + ncof= (n==2)?4:n; + const FLOAT *tmpcc ; + cc.resize(ncof+1) ; + cr.resize(ncof+1) ; + + if (n == 2) + { + tmpcc=c2; + cc[1] = tmpcc[1] ; + cc[2] = tmpcc[2] ; + cc[3] = 0.0f ; + cc[4] = 0.0f ; + cr[1] = tmpcc[1] ; + cr[2] =-tmpcc[2] ; + cr[3] = 0.0f ; + cr[4] = 0.0f ; + + ioff = joff = -1 ; + } + else + { + if (n == 4) + tmpcc=c4; + else if (n == 12) + tmpcc=c12; + else if (n == 20) + tmpcc=c20; + else + throw std::runtime_error("unimplemented value n in pwtset"); + + for (k=1;k<=n;k++) + { + cc[k] = tmpcc[k] ; + cr[ncof+1-k]=sig*tmpcc[k]; + sig = -sig; + } + ioff = joff = -(n >> 1); + } + } + + ~wavefilt() {} + + int ncof,ioff,joff; + std::vector cc; + std::vector cr; + } ; + + static const wavefilt& waveletFilter(WaveletType type) + { + static wavefilt *daub02filt = NULL ; + static wavefilt *daub04filt = NULL ; + static wavefilt *daub12filt = NULL ; + static wavefilt *daub20filt = NULL ; + + switch(type) + { + case DWT_DAUB02: if(daub02filt == NULL) + daub02filt = new wavefilt(2) ; + return *daub02filt ; + + case DWT_DAUB04: if(daub04filt == NULL) + daub04filt = new wavefilt(4) ; + return *daub04filt ; + + case DWT_DAUB12: if(daub12filt == NULL) + daub12filt = new wavefilt(12) ; + return *daub12filt ; + + case DWT_DAUB20: if(daub20filt == NULL) + daub20filt = new wavefilt(20) ; + return *daub20filt ; + + default: + throw std::runtime_error("Unknown wavelet type.") ; + } + } + + static void pwt(FLOAT a[], unsigned long n, int isign,const wavefilt& wfilt) + { +/********************** BEGIN SIGNED PART *************************/ +/** md5sum = 2b9e1e38ac690f50806873cdb4a061ea **/ +/** Validation date = 08/10/10 **/ +/******************************************************************/ + unsigned long i,ii,ni,nj ; + + if (n < 4) + return; + + FLOAT *wksp=new FLOAT[n+1];//vector(1,n); + FLOAT ai,ai1 ; + unsigned long int nmod=wfilt.ncof*n; + unsigned long int n1=n-1; + unsigned long int nh=n >> 1; + + memset(wksp,0,(n+1)*sizeof(FLOAT)) ; + + if (isign == DWT_FORWARD) + for (ii=1,i=1;i<=n;i+=2,ii++) + { + ni=i+nmod+wfilt.ioff; + nj=i+nmod+wfilt.joff; + +#ifdef USE_SSE_INSTRUCTIONS +#warning Using SSE2 Instruction set for wavelet internal loops + for (int k=1;k<=wfilt.ncof;k+=4) + { + int jf=ni+k; + int jr=nj+k; + + sse_block w1(wfilt.cc[k],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) ; + sse_block w2(wfilt.cr[k],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) ; + + sse_block a1( a[1+((jf+0)&n1)], a[1+((jf+1)&n1)], a[1+((jf+2)&n1)], a[1+((jf+3)&n1)]) ; + sse_block a2( a[1+((jr+0)&n1)], a[1+((jr+1)&n1)], a[1+((jr+2)&n1)], a[1+((jr+3)&n1)]) ; + + sse_block wk1( w1*a1 ) ; + sse_block wk2( w2*a2 ) ; + + wksp[ii ] += wk1.sum() ; + wksp[ii+nh] += wk2.sum() ; + } +#else + for (int k=1;k<=wfilt.ncof;k++) + { + int jf=n1 & (ni+k); + int jr=n1 & (nj+k); + wksp[ii] += wfilt.cc[k]*a[jf+1]; + wksp[ii+nh] += wfilt.cr[k]*a[jr+1]; + } +#endif + } + else + for (ii=1,i=1;i<=n;i+=2,ii++) + { + ai=a[ii]; + ai1=a[ii+nh]; + ni=i+nmod+wfilt.ioff; + nj=i+nmod+wfilt.joff; + +#ifdef USE_SSE_INSTRUCTIONS + sse_block ai_sse( ai,ai,ai,ai ) ; + sse_block ai1_sse( ai1,ai1,ai1,ai1 ) ; + + for (int k=1;k<=wfilt.ncof;k+=4) + { + int jf=ni+k ; + int jr=nj+k ; // in fact we have jf==jr, so the code is simpler. + + sse_block w1(wksp[1+((jf+0) & n1)],wksp[1+((jf+1) & n1)],wksp[1+((jf+2) & n1)],wksp[1+((jf+3) & n1)]) ; + + w1 += sse_block(wfilt.cc[k+0],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) * ai_sse ; + w1 += sse_block(wfilt.cr[k+0],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) * ai1_sse ; + + wksp[1+((jr+0) & n1)] = w1[0] ; + wksp[1+((jr+1) & n1)] = w1[1] ; + wksp[1+((jr+2) & n1)] = w1[2] ; + wksp[1+((jr+3) & n1)] = w1[3] ; + } +#else + for (int k=1;k<=wfilt.ncof;++k) + { + wksp[(n1 & (ni+k))+1] += wfilt.cc[k]*ai; + wksp[(n1 & (nj+k))+1] += wfilt.cr[k]*ai1; + } +#endif + } + + for (uint j=1;j<=n;j++) + a[j]=wksp[j]; + + delete[] wksp ;//free_vector(wksp,1,n); +/********************** END SIGNED PART *************************/ + } + + static void wtn(FLOAT a[], unsigned long nn[], int ndim, int isign, const wavefilt& w,void (*wtstep)(FLOAT [], unsigned long, int,const wavefilt&)) + { + unsigned long i1,i2,i3,k,n,nnew,nprev=1,nt,ntot=1; + int idim; + FLOAT *wksp; + + for (idim=1;idim<=ndim;idim++) + ntot *= nn[idim]; + + wksp=new FLOAT[ntot+1] ; //vector(1,ntot); + + for (idim=1;idim<=ndim;idim++) + { + n=nn[idim]; + nnew=n*nprev; + + if (n > 4) + for (i2=0;i2=4;nt >>= 1) + (*wtstep)(wksp,nt,isign,w); + else + for(nt=4;nt<=n;nt <<= 1) + (*wtstep)(wksp,nt,isign,w); + + for (i3=i1+i2,k=1;k<=n;k++,i3+=nprev) a[i3]=wksp[k]; + } + + nprev=nnew; + } + delete[] wksp ;//free_vector(wksp,1,ntot); + } +}; + diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 4e1979a5d..9f4ac7a7c 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -7,12 +7,14 @@ #include "VideoProcessor.h" #include "QVideoDevice.h" +#include "DaubechyWavelets.h" VideoProcessor::VideoProcessor() :_encoded_frame_size(128,128) { _decoded_output_device = NULL ; - _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; + _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; } bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) @@ -31,7 +33,6 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& // std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; - void *data = NULL; encoded_size = 0 ; if(codec) @@ -111,6 +112,8 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) if(codec != NULL) codec->decodeData(chunk,img) ; + else + std::cerr << "Unknown decoding codec: " << codid << std::endl; if(_decoded_output_device) _decoded_output_device->showFrame(img) ; @@ -159,7 +162,7 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) QImage res = _decoded_reference_frame ; - for(uint32_t i=0;i> 8) & 0xff ; - memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],qb.data(),qb.size()) ; voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; @@ -230,3 +233,230 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa return true ; } + +bool WaveletVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +{ + static const int WAVELET_IMG_SIZE = 128 ; + static const float W_THRESHOLD = 0.005 ; // low quality + //static const float W_THRESHOLD = 0.0001; // high quality + //static const float W_THRESHOLD = 0.0005; // medium quality + + static const int W2 = WAVELET_IMG_SIZE ; + static const int H2 = WAVELET_IMG_SIZE ; + + assert(image.width() == W2) ; + assert(image.height() == H2) ; + + float *temp = new float[W2*H2] ; + + std::cerr << " codec type: wavelets." << std::endl; + + // We should perform some interpolation here ;-) + // + for(int i=0;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_FORWARD) ; + + // Now estimate the max energy in the W coefs, and only keep the largest. + + float mx = 0.0f ; + for(int i=0;i compressed_values ; + compressed_values.reserve(W2*H2) ; + + for(int i=0;i= W_THRESHOLD*mx) // This needs to be improved. Wavelets do not all have the same visual impact. + { + // add one value, using 16 bits for coordinates and 16 bits for the value. + + compressed_values.push_back((uint16_t)i) ; + compressed_values.push_back(quantize_16b(temp[i],mx)) ; + + //float f2 = from_quantized_16b(quantize_16b(temp[i],mx),mx) ; + + //if(fabs(f2 - temp[i]) >= 0.01*(fabs(temp[i])+fabs(f2))) + //std::cerr << " before: " << temp[i] << ", quantised=" << quantize_16b(temp[i],mx)<< ", after: " << f2 << std::endl; + } + delete[] temp ; + + // Serialise all values into a memory buffer. This needs to be taken care of because of endian issues. + + int compressed_size = 4 + compressed_values.size()*2 ; + + std::cerr << " threshold : " << W_THRESHOLD << std::endl; + std::cerr << " values kept: " << compressed_values.size()/2 << std::endl; + std::cerr << " compression: " << compressed_size/float(W2*H2*3)*100 << " %" << std::endl; + + voip_chunk.data = malloc(HEADER_SIZE + compressed_size) ; + + // build header + uint32_t flags = 0 ; + + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + + unsigned char *compressed_mem = &((unsigned char *)voip_chunk.data)[HEADER_SIZE] ; + serialise_ufloat(compressed_mem,mx) ; + + for(uint32_t i=0;i> 8 ; + } + + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + voip_chunk.size = HEADER_SIZE + compressed_size ; + + return true ; +} + +bool WaveletVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) +{ + static const int WAVELET_IMG_SIZE = 128 ; + + static const int W2 = WAVELET_IMG_SIZE ; + static const int H2 = WAVELET_IMG_SIZE ; + + float *temp = new float[W2*H2] ; + + const unsigned char *compressed_mem = &static_cast(chunk.data)[HEADER_SIZE] ; + int compressed_size = chunk.size - HEADER_SIZE; + + memset(temp,0,W2*H2*sizeof(float)) ; + float M = deserialise_ufloat(compressed_mem); + +#ifdef VOIP_CODEC_DEBUG + std::cerr << " codec type: wavelets." << std::endl; + std::cerr << " max coef: " << M << std::endl; +#endif + + for(int i=4;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_BACKWARD) ; + +#ifdef VOIP_CODEC_DEBUG + std::cerr << " resizing image to: " << w << "x" << h << std::endl; +#endif + + image = QImage(W2,H2,QImage::Format_RGB32) ; + + int indx = 0 ; + + for(int j=0;j1023), and p is coded on 6 bits (0->63). + // Packing [mp] into a 16bit uint16_t. M is the maximum coefficient over the quantization + // process. + // + // So this represents numbers from M * 1 * 2^{-73} to M + // + // All calculatoins are performed on x/M*2^10 + // + static const float LOG2 = log(2.0f) ; + + int m,p ; + + if(fabs(x) < 1e-8*M) + { + m = 0 ; + p = 0 ; + } + else + { + float log2f = log(fabsf(x)/M)/LOG2 ; + int mexp = (int)floor(MANTISSE_BITS - log2f) ; + + m = (int)floor(pow(2.0f,mexp+log2f)) ; + p = mexp ; + + if(p > (1<> EXPONENT_BITS ; + + if(p > 10) + return M * m / 1024.0f / (float)(1 << (p-10)) ; + else + return M * m / (float)(1 << p) ; +} + +void WaveletVideo::serialise_ufloat(unsigned char *mem, float f) +{ + if(f < 0.0f) + { + std::cerr << "(EE) Cannot serialise invalid negative float value " << f << " in " << __PRETTY_FUNCTION__ << std::endl; + return ; + } + // This serialisation is quite accurate. The max relative error is approx. + // 0.01% and most of the time less than 1e-05% The error is well distributed + // over numbers also. + // + uint32_t n = (f < 1e-7)?(~(uint32_t)0): ((uint32_t)( (1.0f/(1.0f+f) * (~(uint32_t)0)))) ; + + mem[0] = n & 0xff ; n >>= 8 ; + mem[1] = n & 0xff ; n >>= 8 ; + mem[2] = n & 0xff ; n >>= 8 ; + mem[3] = n & 0xff ; +} +float WaveletVideo::deserialise_ufloat(const unsigned char *mem) +{ + uint32_t n = mem[3] ; + n = (n << 8) + mem[2] ; + n = (n << 8) + mem[1] ; + n = (n << 8) + mem[0] ; + + return 1.0f/ ( n/(float)(~(uint32_t)0)) - 1.0f ; +} diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index d103207af..da12d76ae 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -11,6 +11,9 @@ class VideoCodec public: virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) = 0; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) = 0; + +protected: + static const uint32_t HEADER_SIZE = 0x04 ; }; // Now derive various image encoding/decoding algorithms. @@ -25,7 +28,6 @@ protected: virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; - static const uint32_t HEADER_SIZE = 0x04 ; static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; private: QImage _decoded_reference_frame ; @@ -35,17 +37,24 @@ private: uint32_t _encoded_ref_frame_count ; }; -class DifferentialWaveletVideo: public VideoCodec +class WaveletVideo: public VideoCodec { public: - DifferentialWaveletVideo() {} + WaveletVideo() {} protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) { return true ; } - virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) { return true ; } - + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: - QImage _last_reference_frame ; + + static const int MANTISSE_BITS = 9 ; + static const int EXPONENT_BITS = 6 ; + + static void serialise_ufloat(unsigned char *mem, float f); + static float deserialise_ufloat(const unsigned char *mem); + + static float from_quantized_16b(uint16_t n, float M); + static uint16_t quantize_16b(float x, float M); }; // This class decodes video from a stream. It keeps a queue of @@ -106,7 +115,7 @@ class VideoProcessor // ===================================================================================== JPEGVideo _jpeg_video_codec ; - DifferentialWaveletVideo _ddwt_video_codec ; + WaveletVideo _ddwt_video_codec ; uint16_t _encoding_current_codec ; }; From 043fe537896458a5d2388eb9486796dcd76b88aa Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 19 Aug 2015 21:50:51 -0400 Subject: [PATCH 09/43] added avcodec encoding --- plugins/VOIP/VOIP.pro | 2 +- plugins/VOIP/gui/VideoProcessor.cpp | 190 +++++++++++++++++++++++++++- plugins/VOIP/gui/VideoProcessor.h | 27 +++- 3 files changed, 211 insertions(+), 8 deletions(-) diff --git a/plugins/VOIP/VOIP.pro b/plugins/VOIP/VOIP.pro index e7f054553..86a716309 100644 --- a/plugins/VOIP/VOIP.pro +++ b/plugins/VOIP/VOIP.pro @@ -96,4 +96,4 @@ TRANSLATIONS += \ lang/VOIP_tr.ts \ lang/VOIP_zh_CN.ts -LIBS += -lspeex -lspeexdsp +LIBS += -lspeex -lspeexdsp -lavformat -lavcodec -lavutil diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 9f4ac7a7c..66a810b0a 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -9,12 +9,26 @@ #include "QVideoDevice.h" #include "DaubechyWavelets.h" +#include + +extern "C" { +#include + +#include +#include +#include +#include +#include +#include +} + VideoProcessor::VideoProcessor() :_encoded_frame_size(128,128) { _decoded_output_device = NULL ; - //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; - _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; + _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; } bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) @@ -27,6 +41,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& break ; case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; break ; + case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; + break ; default: codec = NULL ; } @@ -95,7 +111,7 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) } uint32_t codid = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; - uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; + //uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; VideoCodec *codec ; @@ -105,6 +121,8 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) break ; case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; break ; + case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; + break ; default: codec = NULL ; } @@ -258,7 +276,7 @@ bool WaveletVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataC std::cerr << " resized image to B&W " << W2 << "x" << H2 << std::endl; - DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_FORWARD) ; + DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_FORWARD) ; // Now estimate the max energy in the W coefs, and only keep the largest. @@ -358,7 +376,7 @@ bool WaveletVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << " values read: " << compressed_size/4-1 << std::endl; #endif - DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB04,DaubechyWavelets::DWT_BACKWARD) ; + DaubechyWavelets::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_BACKWARD) ; #ifdef VOIP_CODEC_DEBUG std::cerr << " resizing image to: " << w << "x" << h << std::endl; @@ -460,3 +478,165 @@ float WaveletVideo::deserialise_ufloat(const unsigned char *mem) return 1.0f/ ( n/(float)(~(uint32_t)0)) - 1.0f ; } + +FFmpegVideo::FFmpegVideo() +{ + codec = NULL ; + frame_buffer = NULL ; + context = NULL ; + + AVCodecID codec_id = AV_CODEC_ID_H264 ; // AV_CODEC_ID_MPEG1VIDEO + + uint8_t endcode[] = { 0, 0, 1, 0xb7 }; + + /* find the mpeg1 video encoder */ + codec = avcodec_find_encoder(codec_id); + + if (!codec) + throw("AV codec not found for codec id ") ; + + context = avcodec_alloc_context3(codec); + + if (!context) + throw std::runtime_error("AV: Could not allocate video codec context"); + + /* put sample parameters */ + context->bit_rate = 400000; + /* resolution must be a multiple of two */ + context->width = 352; + context->height = 288; + /* frames per second */ + context->time_base = (AVRational){1,25}; + /* emit one intra frame every ten frames + * check frame pict_type before passing frame + * to encoder, if frame->pict_type is AV_PICTURE_TYPE_I + * then gop_size is ignored and the output of encoder + * will always be I frame irrespective to gop_size + */ + context->gop_size = 10; + context->max_b_frames = 1; + //context->pix_fmt = AV_PIX_FMT_RGB24; + context->pix_fmt = AV_PIX_FMT_YUV420P; + + if (codec_id == AV_CODEC_ID_H264) + av_opt_set(context->priv_data, "preset", "slow", 0); + + /* open it */ + if (avcodec_open2(context, codec, NULL) < 0) + throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); + + frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; + + frame_buffer->format = context->pix_fmt; + frame_buffer->width = context->width; + frame_buffer->height = context->height; + + /* the image can be allocated by any means and av_image_alloc() is + * just the most convenient way if av_malloc() is to be used */ + + int ret = av_image_alloc(frame_buffer->data, frame_buffer->linesize, context->width, context->height, context->pix_fmt, 32); + + if (ret < 0) + throw std::runtime_error("AV: Could not allocate raw picture buffer"); + + frame_count = 0 ; +} + +FFmpegVideo::~FFmpegVideo() +{ + avcodec_close(context); + + av_free(context); + + av_freep(&frame_buffer->data[0]); + + free(frame_buffer); +} + + +bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +{ + AVPacket pkt ; + av_init_packet(&pkt); + pkt.data = NULL; // packet data will be allocated by the encoder + pkt.size = 0; + + QImage input ; + + if(image.width() != frame_buffer->width || image.height() != frame_buffer->height) + input = image.scaled(QSize(frame_buffer->width,frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + else + input = image ; + + /* prepare a dummy image */ + /* Y */ + for (int y = 0; y < context->height; y++) + for (int x = 0; x < context->width; x++) + { + QRgb pix = image.pixel(QPoint(x,y)) ; + + register int R = (pix >> 16) & 0xff ; + register int G = (pix >> 8) & 0xff ; + register int B = (pix >> 0) & 0xff ; + + register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; + register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; + + frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y + frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr + frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + } + + + frame_buffer->pts = frame_count++; + + /* encode the image */ + + int got_output = 0; + + AVFrame *frame = frame_buffer ; + +// do +// { + int ret = avcodec_encode_video2(context, &pkt, frame, &got_output) ; + + if (ret < 0) + { + std::cerr << "Error encoding frame!" << std::endl; + return false ; + } +// frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer +// +// } while(got_output) ; + + if(got_output) + { + voip_chunk.data = pkt.data ; + voip_chunk.size = pkt.size ; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + } + else + { + voip_chunk.data = NULL; + voip_chunk.size = 0; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "No output produced." << std::endl; + } + + + pkt.data = NULL ; // transfer ownership to chunk + pkt.size = 0 ; + + av_free_packet(&pkt); + + return true ; +} +bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) +{ + return true ; +} + diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index da12d76ae..7cc7c1526 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -57,6 +57,27 @@ private: static uint16_t quantize_16b(float x, float M); }; +struct AVCodec ; +struct AVCodecContext ; +struct AVFrame ; + +class FFmpegVideo: public VideoCodec +{ +public: + FFmpegVideo() ; + ~FFmpegVideo() ; + +protected: + virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; + +private: + AVCodec *codec; + AVCodecContext *context; + AVFrame *frame_buffer ; + uint64_t frame_count ; +}; + // This class decodes video from a stream. It keeps a queue of // decoded frame that needs to be retrieved using the getNextImage() method. // @@ -69,7 +90,8 @@ class VideoProcessor enum CodecId { VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000, VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO = 0x0001, - VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002 + VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002, + VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO = 0x0003 }; // ===================================================================================== @@ -114,8 +136,9 @@ class VideoProcessor // =------------------------------------- Codecs --------------------------------------= // ===================================================================================== - JPEGVideo _jpeg_video_codec ; + JPEGVideo _jpeg_video_codec ; WaveletVideo _ddwt_video_codec ; + FFmpegVideo _mpeg_video_codec ; uint16_t _encoding_current_codec ; }; From 9798c78e2091b1edd6a7be7427aea707dbd67bc2 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 19 Aug 2015 22:17:50 -0400 Subject: [PATCH 10/43] added decoding of avcodec --- plugins/VOIP/gui/VideoProcessor.cpp | 237 ++++++++++++++++------------ plugins/VOIP/gui/VideoProcessor.h | 11 +- 2 files changed, 146 insertions(+), 102 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 66a810b0a..97ef75271 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -481,162 +481,203 @@ float WaveletVideo::deserialise_ufloat(const unsigned char *mem) FFmpegVideo::FFmpegVideo() { - codec = NULL ; - frame_buffer = NULL ; - context = NULL ; + // Encoding + + encoding_codec = NULL ; + encoding_frame_buffer = NULL ; + encoding_context = NULL ; AVCodecID codec_id = AV_CODEC_ID_H264 ; // AV_CODEC_ID_MPEG1VIDEO uint8_t endcode[] = { 0, 0, 1, 0xb7 }; /* find the mpeg1 video encoder */ - codec = avcodec_find_encoder(codec_id); + encoding_codec = avcodec_find_encoder(codec_id); - if (!codec) - throw("AV codec not found for codec id ") ; + if (!encoding_codec) throw("AV codec not found for codec id ") ; - context = avcodec_alloc_context3(codec); + encoding_context = avcodec_alloc_context3(encoding_codec); - if (!context) - throw std::runtime_error("AV: Could not allocate video codec context"); + if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); + if (!decoding_context) throw std::runtime_error("AV: Could not allocate video codec decoding context"); /* put sample parameters */ - context->bit_rate = 400000; + encoding_context->bit_rate = 400000; /* resolution must be a multiple of two */ - context->width = 352; - context->height = 288; + encoding_context->width = 352; + encoding_context->height = 288; /* frames per second */ - context->time_base = (AVRational){1,25}; + encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames * check frame pict_type before passing frame * to encoder, if frame->pict_type is AV_PICTURE_TYPE_I * then gop_size is ignored and the output of encoder * will always be I frame irrespective to gop_size */ - context->gop_size = 10; - context->max_b_frames = 1; + encoding_context->gop_size = 10; + encoding_context->max_b_frames = 1; //context->pix_fmt = AV_PIX_FMT_RGB24; - context->pix_fmt = AV_PIX_FMT_YUV420P; + encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; if (codec_id == AV_CODEC_ID_H264) - av_opt_set(context->priv_data, "preset", "slow", 0); + av_opt_set(encoding_context->priv_data, "preset", "slow", 0); /* open it */ - if (avcodec_open2(context, codec, NULL) < 0) + if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0) throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); - frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; + encoding_frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; - frame_buffer->format = context->pix_fmt; - frame_buffer->width = context->width; - frame_buffer->height = context->height; + encoding_frame_buffer->format = encoding_context->pix_fmt; + encoding_frame_buffer->width = encoding_context->width; + encoding_frame_buffer->height = encoding_context->height; /* the image can be allocated by any means and av_image_alloc() is * just the most convenient way if av_malloc() is to be used */ - int ret = av_image_alloc(frame_buffer->data, frame_buffer->linesize, context->width, context->height, context->pix_fmt, 32); + int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); if (ret < 0) throw std::runtime_error("AV: Could not allocate raw picture buffer"); - frame_count = 0 ; + encoding_frame_count = 0 ; + + // Decoding + + + decoding_codec = avcodec_find_decoder(codec_id); + + if (!decoding_codec) + throw("AV codec not found for codec id ") ; + + decoding_context = avcodec_alloc_context3(decoding_codec); + } FFmpegVideo::~FFmpegVideo() { - avcodec_close(context); - - av_free(context); - - av_freep(&frame_buffer->data[0]); - - free(frame_buffer); + avcodec_close(encoding_context); + avcodec_close(decoding_context); + av_free(encoding_context); + av_free(decoding_context); + av_freep(&encoding_frame_buffer->data[0]); + av_freep(&decoding_frame_buffer->data[0]); + free(encoding_frame_buffer); + free(decoding_frame_buffer); } bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - AVPacket pkt ; - av_init_packet(&pkt); - pkt.data = NULL; // packet data will be allocated by the encoder - pkt.size = 0; + AVPacket pkt ; + av_init_packet(&pkt); + pkt.data = NULL; // packet data will be allocated by the encoder + pkt.size = 0; - QImage input ; + QImage input ; - if(image.width() != frame_buffer->width || image.height() != frame_buffer->height) - input = image.scaled(QSize(frame_buffer->width,frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; - else - input = image ; + if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) + input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; + else + input = image ; - /* prepare a dummy image */ - /* Y */ - for (int y = 0; y < context->height; y++) - for (int x = 0; x < context->width; x++) + /* prepare a dummy image */ + /* Y */ + for (int y = 0; y < encoding_context->height; y++) + for (int x = 0; x < encoding_context->width; x++) + { + QRgb pix = image.pixel(QPoint(x,y)) ; + + register int R = (pix >> 16) & 0xff ; + register int G = (pix >> 8) & 0xff ; + register int B = (pix >> 0) & 0xff ; + + register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; + register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; + + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + } + + + encoding_frame_buffer->pts = encoding_frame_count++; + + /* encode the image */ + + int got_output = 0; + + AVFrame *frame = encoding_frame_buffer ; + + // do + // { + int ret = avcodec_encode_video2(encoding_context, &pkt, frame, &got_output) ; + + if (ret < 0) { - QRgb pix = image.pixel(QPoint(x,y)) ; + std::cerr << "Error encoding frame!" << std::endl; + return false ; + } + // frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer + // + // } while(got_output) ; - register int R = (pix >> 16) & 0xff ; - register int G = (pix >> 8) & 0xff ; - register int B = (pix >> 0) & 0xff ; + if(got_output) + { + voip_chunk.data = pkt.data ; + voip_chunk.size = pkt.size ; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; - register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; - register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; - - frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y - frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr - frame_buffer->data[0][y * frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + } + else + { + voip_chunk.data = NULL; + voip_chunk.size = 0; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "No output produced." << std::endl; } - frame_buffer->pts = frame_count++; + pkt.data = NULL ; // transfer ownership to chunk + pkt.size = 0 ; - /* encode the image */ + av_free_packet(&pkt); - int got_output = 0; - - AVFrame *frame = frame_buffer ; - -// do -// { - int ret = avcodec_encode_video2(context, &pkt, frame, &got_output) ; - - if (ret < 0) - { - std::cerr << "Error encoding frame!" << std::endl; - return false ; - } -// frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer -// -// } while(got_output) ; - - if(got_output) - { - voip_chunk.data = pkt.data ; - voip_chunk.size = pkt.size ; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; - } - else - { - voip_chunk.data = NULL; - voip_chunk.size = 0; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - std::cerr << "No output produced." << std::endl; - } - - - pkt.data = NULL ; // transfer ownership to chunk - pkt.size = 0 ; - - av_free_packet(&pkt); - - return true ; + return true ; } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { + AVPacket pkt ; + + av_init_packet(&pkt); + pkt.data = static_cast(chunk.data); // packet data will be allocated by the encoder + pkt.size = chunk.size; + + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + + if(!got_frame) + return true; + + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int V = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + + register int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + register int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + register int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128) +))) ; + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + return true ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 7cc7c1526..c60fc9864 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -72,10 +72,13 @@ protected: virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: - AVCodec *codec; - AVCodecContext *context; - AVFrame *frame_buffer ; - uint64_t frame_count ; + AVCodec *encoding_codec; + AVCodec *decoding_codec; + AVCodecContext *encoding_context; + AVCodecContext *decoding_context; + AVFrame *encoding_frame_buffer ; + AVFrame *decoding_frame_buffer ; + uint64_t encoding_frame_count ; }; // This class decodes video from a stream. It keeps a queue of From 1249423eff6c484abda171822b7c5d46b6e6c84e Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 20 Aug 2015 21:46:28 -0400 Subject: [PATCH 11/43] worked out some bugs. Still not working --- plugins/VOIP/gui/VideoProcessor.cpp | 68 ++++++++++++++++++++--------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 97ef75271..9f4ebec6e 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -128,10 +129,17 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) } QImage img ; - if(codec != NULL) - codec->decodeData(chunk,img) ; - else + if(codec == NULL) + { std::cerr << "Unknown decoding codec: " << codid << std::endl; + return ; + } + + if(!codec->decodeData(chunk,img)) + { + std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; + return ; + } if(_decoded_output_device) _decoded_output_device->showFrame(img) ; @@ -487,7 +495,8 @@ FFmpegVideo::FFmpegVideo() encoding_frame_buffer = NULL ; encoding_context = NULL ; - AVCodecID codec_id = AV_CODEC_ID_H264 ; // AV_CODEC_ID_MPEG1VIDEO + //AVCodecID codec_id = AV_CODEC_ID_H264 ; + AVCodecID codec_id = AV_CODEC_ID_MPEG1VIDEO; uint8_t endcode[] = { 0, 0, 1, 0xb7 }; @@ -526,7 +535,7 @@ FFmpegVideo::FFmpegVideo() if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0) throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); - encoding_frame_buffer = (AVFrame*)malloc(sizeof(AVFrame)) ; + encoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; encoding_frame_buffer->format = encoding_context->pix_fmt; encoding_frame_buffer->width = encoding_context->width; @@ -544,7 +553,7 @@ FFmpegVideo::FFmpegVideo() // Decoding - + decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; decoding_codec = avcodec_find_decoder(codec_id); if (!decoding_codec) @@ -552,6 +561,11 @@ FFmpegVideo::FFmpegVideo() decoding_context = avcodec_alloc_context3(decoding_codec); +// if (decoding_codec->capabilities & AV_CODEC_CAP_TRUNCATED) +// decoding_context->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames + + if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) + throw("AV codec open action failed! ") ; } FFmpegVideo::~FFmpegVideo() @@ -574,6 +588,7 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh pkt.data = NULL; // packet data will be allocated by the encoder pkt.size = 0; + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -624,13 +639,22 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh // } while(got_output) ; if(got_output) - { - voip_chunk.data = pkt.data ; - voip_chunk.size = pkt.size ; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + { + voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; + uint32_t flags = 0; - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; - } + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ; + + voip_chunk.size = pkt.size + HEADER_SIZE; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + } else { voip_chunk.data = NULL; @@ -640,10 +664,6 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh std::cerr << "No output produced." << std::endl; } - - pkt.data = NULL ; // transfer ownership to chunk - pkt.size = 0 ; - av_free_packet(&pkt); return true ; @@ -653,23 +673,29 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) AVPacket pkt ; av_init_packet(&pkt); - pkt.data = static_cast(chunk.data); // packet data will be allocated by the encoder - pkt.size = chunk.size; + pkt.data = (unsigned char *)memalign(16,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + + pkt.size = chunk.size - HEADER_SIZE; + int got_frame = 0 ; int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; if(!got_frame) - return true; + return false; image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; + for (int y = 0; y < decoding_frame_buffer->height; y++) for (int x = 0; x < decoding_frame_buffer->width; x++) { int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int V = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[0] + x] ; + int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[0] + x] ; register int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; register int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; From 4522fe85cc57e72e505eb4dd889b400f7970d5cd Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 20 Aug 2015 22:59:51 -0400 Subject: [PATCH 12/43] fixed encoding --- plugins/VOIP/gui/VideoProcessor.cpp | 102 ++++++++++++++++------------ plugins/VOIP/gui/VideoProcessor.h | 2 + 2 files changed, 60 insertions(+), 44 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 9f4ebec6e..6faab4c80 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -22,9 +22,10 @@ extern "C" { #include #include } +#define DEBUG_MPEG_VIDEO 1 VideoProcessor::VideoProcessor() - :_encoded_frame_size(128,128) + :_encoded_frame_size(176,144) { _decoded_output_device = NULL ; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; @@ -135,14 +136,14 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } - if(!codec->decodeData(chunk,img)) - { - std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; - return ; - } - - if(_decoded_output_device) - _decoded_output_device->showFrame(img) ; +// if(!codec->decodeData(chunk,img)) +// { +// std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; +// return ; +// } +// +// if(_decoded_output_device) +// _decoded_output_device->showFrame(img) ; } void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) @@ -513,8 +514,8 @@ FFmpegVideo::FFmpegVideo() /* put sample parameters */ encoding_context->bit_rate = 400000; /* resolution must be a multiple of two */ - encoding_context->width = 352; - encoding_context->height = 288; + encoding_context->width = 176; + encoding_context->height = 144; /* frames per second */ encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames @@ -560,12 +561,21 @@ FFmpegVideo::FFmpegVideo() throw("AV codec not found for codec id ") ; decoding_context = avcodec_alloc_context3(decoding_codec); + decoding_context->width = encoding_context->width; + decoding_context->height = encoding_context->height; + decoding_context->pix_fmt = AV_PIX_FMT_YUV420P; // if (decoding_codec->capabilities & AV_CODEC_CAP_TRUNCATED) // decoding_context->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) throw("AV codec open action failed! ") ; + + // debug +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Dumping captured data to file tmpvideo.mpg" << std::endl; + encoding_debug_file = fopen("tmpvideo.mpg","w") ; +#endif } FFmpegVideo::~FFmpegVideo() @@ -601,19 +611,19 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh for (int y = 0; y < encoding_context->height; y++) for (int x = 0; x < encoding_context->width; x++) { - QRgb pix = image.pixel(QPoint(x,y)) ; + QRgb pix = input.pixel(QPoint(x,y)) ; register int R = (pix >> 16) & 0xff ; register int G = (pix >> 8) & 0xff ; register int B = (pix >> 0) & 0xff ; - register int y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; - register int u = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; - register int v = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; + int Y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + int U = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; + int V = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,y)); // Y - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,u));// Cr - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,v));// Cb + encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,Y)); // Y + encoding_frame_buffer->data[1][y * encoding_frame_buffer->linesize[1] + x] = std::min(255,std::max(0,U));// Cr + encoding_frame_buffer->data[2][y * encoding_frame_buffer->linesize[2] + x] = std::min(255,std::max(0,V));// Cb } @@ -654,6 +664,10 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; std::cerr << "Output : " << pkt.size << " bytes." << std::endl; +#ifdef DEBUG_MPEG_VIDEO + fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; + fflush(encoding_debug_file) ; +#endif } else { @@ -670,37 +684,37 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { - AVPacket pkt ; - - av_init_packet(&pkt); - - pkt.data = (unsigned char *)memalign(16,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; - - pkt.size = chunk.size - HEADER_SIZE; + AVPacket pkt ; - int got_frame = 0 ; - int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; - - if(!got_frame) - return false; - - image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; - - std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; - - for (int y = 0; y < decoding_frame_buffer->height; y++) + av_init_packet(&pkt); + + pkt.data = (unsigned char *)memalign(32,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + + pkt.size = chunk.size - HEADER_SIZE; + + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + + if(!got_frame) + return false; + + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + + std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; + + for (int y = 0; y < decoding_frame_buffer->height; y++) for (int x = 0; x < decoding_frame_buffer->width; x++) { int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[0] + x] ; - int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; + int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; - register int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; - register int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - register int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128) -))) ; image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index c60fc9864..b947c1a0a 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -79,6 +79,8 @@ private: AVFrame *encoding_frame_buffer ; AVFrame *decoding_frame_buffer ; uint64_t encoding_frame_count ; + + FILE *encoding_debug_file ; }; // This class decodes video from a stream. It keeps a queue of From 0bb5a74099d28da1687bf8b688d6e1dc350099dd Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 10:27:50 -0400 Subject: [PATCH 13/43] fixed some bugs. Still crashing in the decoding phase. Don_t know why. --- plugins/VOIP/gui/VideoProcessor.cpp | 110 ++++++++++++++++------------ 1 file changed, 65 insertions(+), 45 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 6faab4c80..ed5436043 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -12,6 +12,7 @@ #include + extern "C" { #include @@ -136,14 +137,14 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } -// if(!codec->decodeData(chunk,img)) -// { -// std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; -// return ; -// } -// -// if(_decoded_output_device) -// _decoded_output_device->showFrame(img) ; + if(!codec->decodeData(chunk,img)) + { + std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; + return ; + } + + if(_decoded_output_device) + _decoded_output_device->showFrame(img) ; } void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) @@ -509,7 +510,6 @@ FFmpegVideo::FFmpegVideo() encoding_context = avcodec_alloc_context3(encoding_codec); if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); - if (!decoding_context) throw std::runtime_error("AV: Could not allocate video codec decoding context"); /* put sample parameters */ encoding_context->bit_rate = 400000; @@ -554,23 +554,32 @@ FFmpegVideo::FFmpegVideo() // Decoding - decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; decoding_codec = avcodec_find_decoder(codec_id); if (!decoding_codec) throw("AV codec not found for codec id ") ; decoding_context = avcodec_alloc_context3(decoding_codec); + + if(!decoding_context) + throw std::runtime_error("AV: Could not allocate video codec decoding context"); + decoding_context->width = encoding_context->width; decoding_context->height = encoding_context->height; decoding_context->pix_fmt = AV_PIX_FMT_YUV420P; -// if (decoding_codec->capabilities & AV_CODEC_CAP_TRUNCATED) -// decoding_context->flags |= AV_CODEC_FLAG_TRUNCATED; // we do not send complete frames + if(decoding_codec->capabilities & CODEC_CAP_TRUNCATED) + decoding_context->flags |= CODEC_FLAG_TRUNCATED; // we do not send complete frames if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) throw("AV codec open action failed! ") ; + decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); + + //if (ret < 0) + //throw std::runtime_error("AV: Could not allocate raw picture buffer"); + // debug #ifdef DEBUG_MPEG_VIDEO std::cerr << "Dumping captured data to file tmpvideo.mpg" << std::endl; @@ -593,11 +602,6 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - AVPacket pkt ; - av_init_packet(&pkt); - pkt.data = NULL; // packet data will be allocated by the encoder - pkt.size = 0; - std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; QImage input ; @@ -635,6 +639,11 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh AVFrame *frame = encoding_frame_buffer ; + AVPacket pkt ; + av_init_packet(&pkt); + pkt.data = NULL; // packet data will be allocated by the encoder + pkt.size = 0; + // do // { int ret = avcodec_encode_video2(encoding_context, &pkt, frame, &got_output) ; @@ -662,12 +671,13 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.size = pkt.size + HEADER_SIZE; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; #ifdef DEBUG_MPEG_VIDEO fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; fflush(encoding_debug_file) ; #endif + av_free_packet(&pkt); } else { @@ -678,46 +688,56 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh std::cerr << "No output produced." << std::endl; } - av_free_packet(&pkt); - return true ; } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { - AVPacket pkt ; + std::cerr << "Decoding data of size " << chunk.size << std::endl; - av_init_packet(&pkt); + AVPacket pkt ; + av_init_packet(&pkt); - pkt.data = (unsigned char *)memalign(32,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memset(pkt.data,0,chunk.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + pkt.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(pkt.data,0,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; + memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; - pkt.size = chunk.size - HEADER_SIZE; + pkt.size = chunk.size - HEADER_SIZE; + pkt.pts = AV_NOPTS_VALUE ; + pkt.dts = AV_NOPTS_VALUE ; - int got_frame = 0 ; - int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; - if(!got_frame) - return false; + if(len < 0) + { + std::cerr << "Error decodign frame!" << std::endl; + return false ; + } + if(!got_frame) + return false; - image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + assert(pkt.size > 0) ; - std::cerr << "Decoding frame. Size=" << image.width() << "x" << image.height() << std::endl; + av_free_packet(&pkt) ; - for (int y = 0; y < decoding_frame_buffer->height; y++) - for (int x = 0; x < decoding_frame_buffer->width; x++) - { - int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; - int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; - int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; - int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; - - image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; - } + std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; - return true ; + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; + int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + + return true ; } From 024999ed670391af62f1f6957c0da8eda46bc2bf Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 21:08:27 -0400 Subject: [PATCH 14/43] fixed bug in creating input image when encoding video --- plugins/VOIP/gui/VideoProcessor.cpp | 91 ++++++++++++++++++----------- 1 file changed, 56 insertions(+), 35 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index ed5436043..aafc092c9 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -58,14 +58,11 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& { RsVOIPDataChunk chunk ; - codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) ; - - encoded_size = chunk.size ; - - if(chunk.size == 0) // the codec might be buffering the frame for compression reasons - return true ; - - _encoded_out_queue.push_back(chunk) ; + if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) + { + encoded_size = chunk.size ; + _encoded_out_queue.push_back(chunk) ; + } return true ; } @@ -538,6 +535,9 @@ FFmpegVideo::FFmpegVideo() encoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + if(!encoding_frame_buffer) + throw std::runtime_error("AV: could not allocate frame buffer.") ; + encoding_frame_buffer->format = encoding_context->pix_fmt; encoding_frame_buffer->width = encoding_context->width; encoding_frame_buffer->height = encoding_context->height; @@ -545,7 +545,8 @@ FFmpegVideo::FFmpegVideo() /* the image can be allocated by any means and av_image_alloc() is * just the most convenient way if av_malloc() is to be used */ - int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); + int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, + encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); if (ret < 0) throw std::runtime_error("AV: Could not allocate raw picture buffer"); @@ -575,6 +576,7 @@ FFmpegVideo::FFmpegVideo() throw("AV codec open action failed! ") ; decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); //if (ret < 0) @@ -602,7 +604,7 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -612,20 +614,36 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh /* prepare a dummy image */ /* Y */ - for (int y = 0; y < encoding_context->height; y++) - for (int x = 0; x < encoding_context->width; x++) + for (int y = 0; y < encoding_context->height/2; y++) + for (int x = 0; x < encoding_context->width/2; x++) { - QRgb pix = input.pixel(QPoint(x,y)) ; + QRgb pix00 = input.pixel(QPoint(2*x+0,2*y+0)) ; + QRgb pix01 = input.pixel(QPoint(2*x+0,2*y+1)) ; + QRgb pix10 = input.pixel(QPoint(2*x+1,2*y+0)) ; + QRgb pix11 = input.pixel(QPoint(2*x+1,2*y+1)) ; - register int R = (pix >> 16) & 0xff ; - register int G = (pix >> 8) & 0xff ; - register int B = (pix >> 0) & 0xff ; + int R00 = (pix00 >> 16) & 0xff ; int G00 = (pix00 >> 8) & 0xff ; int B00 = (pix00 >> 0) & 0xff ; + int R01 = (pix01 >> 16) & 0xff ; int G01 = (pix01 >> 8) & 0xff ; int B01 = (pix01 >> 0) & 0xff ; + int R10 = (pix10 >> 16) & 0xff ; int G10 = (pix10 >> 8) & 0xff ; int B10 = (pix10 >> 0) & 0xff ; + int R11 = (pix11 >> 16) & 0xff ; int G11 = (pix11 >> 8) & 0xff ; int B11 = (pix11 >> 0) & 0xff ; - int Y = (0.257 * R) + (0.504 * G) + (0.098 * B) + 16 ; + int Y00 = (0.257 * R00) + (0.504 * G00) + (0.098 * B00) + 16 ; + int Y01 = (0.257 * R01) + (0.504 * G01) + (0.098 * B01) + 16 ; + int Y10 = (0.257 * R10) + (0.504 * G10) + (0.098 * B10) + 16 ; + int Y11 = (0.257 * R11) + (0.504 * G11) + (0.098 * B11) + 16 ; + + float R = 0.25*(R00+R01+R10+R11) ; + float G = 0.25*(G00+G01+G10+G11) ; + float B = 0.25*(B00+B01+B10+B11) ; + int U = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; int V = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; - encoding_frame_buffer->data[0][y * encoding_frame_buffer->linesize[0] + x] = std::min(255,std::max(0,Y)); // Y + encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y00)); // Y + encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y01)); // Y + encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y10)); // Y + encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y11)); // Y + encoding_frame_buffer->data[1][y * encoding_frame_buffer->linesize[1] + x] = std::min(255,std::max(0,U));// Cr encoding_frame_buffer->data[2][y * encoding_frame_buffer->linesize[2] + x] = std::min(255,std::max(0,V));// Cb } @@ -658,27 +676,29 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh // } while(got_output) ; if(got_output) - { - voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; - uint32_t flags = 0; + { + voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; + uint32_t flags = 0; - ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; - ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ; - ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; - ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; + ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ; + ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; + ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; - memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ; + memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ; - voip_chunk.size = pkt.size + HEADER_SIZE; - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; + voip_chunk.size = pkt.size + HEADER_SIZE; + voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; + + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; #ifdef DEBUG_MPEG_VIDEO - fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; - fflush(encoding_debug_file) ; + fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; + fflush(encoding_debug_file) ; #endif - av_free_packet(&pkt); - } + av_free_packet(&pkt); + + return true ; + } else { voip_chunk.data = NULL; @@ -686,9 +706,9 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; std::cerr << "No output produced." << std::endl; + return false ; } - return true ; } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { @@ -713,6 +733,7 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << "Error decodign frame!" << std::endl; return false ; } + std::cerr << "Used " << len << " bytes out of " << pkt.size << ". got_frame = " << got_frame << std::endl; if(!got_frame) return false; From f55c115bdbbc7540376e87ee939076eb340ac8b3 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 22:39:16 -0400 Subject: [PATCH 15/43] fixed a few bugs. Still not working --- plugins/VOIP/gui/QVideoDevice.cpp | 5 +- plugins/VOIP/gui/VideoProcessor.cpp | 107 ++++++++++++++++++---------- plugins/VOIP/gui/VideoProcessor.h | 6 ++ 3 files changed, 81 insertions(+), 37 deletions(-) diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index e9ace94f9..c3deac879 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -83,8 +83,9 @@ void QVideoInputDevice::grabFrame() uint32_t encoded_size ; _video_processor->processImage(image,0,encoded_size) ; - +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE std::cerr << "Encoded size = " << encoded_size << std::endl; +#endif _total_encoded_size += encoded_size ; time_t now = time(NULL) ; @@ -96,7 +97,9 @@ void QVideoInputDevice::grabFrame() _total_encoded_size = 0 ; _last_bw_estimate_TS = now ; +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE std::cerr << "new bw estimate: " << _estimated_bw << std::endl; +#endif } emit networkPacketReady() ; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index aafc092c9..cadef3048 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -577,6 +577,10 @@ FFmpegVideo::FFmpegVideo() decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + av_init_packet(&decoding_buffer); + decoding_buffer.data = NULL ; + decoding_buffer.size = 0 ; + //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); //if (ret < 0) @@ -604,7 +608,7 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { - std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << std::endl; + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -712,53 +716,84 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { - std::cerr << "Decoding data of size " << chunk.size << std::endl; + std::cerr << "Decoding data of size " << chunk.size << std::endl; + unsigned char *buff ; - AVPacket pkt ; - av_init_packet(&pkt); + if(decoding_buffer.data != NULL) + { + std::cerr << "Completing buffer with size " << chunk.size - HEADER_SIZE + decoding_buffer.size << ": copying existing " + << decoding_buffer.size << " bytes. Adding new " << chunk.size - HEADER_SIZE<< " bytes " << std::endl; - pkt.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; - memset(pkt.data,0,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; - memcpy(pkt.data,&static_cast(chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + uint32_t s = chunk.size - HEADER_SIZE + decoding_buffer.size ; + unsigned char *tmp = (unsigned char*)memalign(16,s + FF_INPUT_BUFFER_PADDING_SIZE) ; + memset(tmp,0,s+FF_INPUT_BUFFER_PADDING_SIZE) ; - pkt.size = chunk.size - HEADER_SIZE; - pkt.pts = AV_NOPTS_VALUE ; - pkt.dts = AV_NOPTS_VALUE ; + memcpy(tmp,decoding_buffer.data,decoding_buffer.size) ; - int got_frame = 0 ; - int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&pkt) ; + free(decoding_buffer.data) ; - if(len < 0) - { - std::cerr << "Error decodign frame!" << std::endl; - return false ; - } - std::cerr << "Used " << len << " bytes out of " << pkt.size << ". got_frame = " << got_frame << std::endl; - if(!got_frame) - return false; + buff = &tmp[decoding_buffer.size] ; + decoding_buffer.size = s ; + decoding_buffer.data = tmp ; + } + else + { + std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl; - assert(pkt.size > 0) ; + decoding_buffer.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; + decoding_buffer.size = chunk.size - HEADER_SIZE ; + memset(decoding_buffer.data,0,decoding_buffer.size + FF_INPUT_BUFFER_PADDING_SIZE) ; - av_free_packet(&pkt) ; + buff = decoding_buffer.data ; + } - image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + memcpy(buff,chunk.data,chunk.size - HEADER_SIZE) ; - std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; + int got_frame = 0 ; + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; - for (int y = 0; y < decoding_frame_buffer->height; y++) - for (int x = 0; x < decoding_frame_buffer->width; x++) - { - int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[1][y * decoding_frame_buffer->linesize[1] + x] ; - int V = decoding_frame_buffer->data[2][y * decoding_frame_buffer->linesize[2] + x] ; + if(len < 0) + { + std::cerr << "Error decodign frame!" << std::endl; + return false ; + } + std::cerr << "Used " << len << " bytes out of " << decoding_buffer.size << ". got_frame = " << got_frame << std::endl; - int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; - int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + if(got_frame) + { + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; - image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; - } + std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; - return true ; + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ; + int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ; + + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + } + + if(len == decoding_buffer.size) + { + free(decoding_buffer.data) ; + decoding_buffer.data = NULL; + decoding_buffer.size = 0; + } + else if(len != 0) + { + std::cerr << "Moving remaining data (" << decoding_buffer.size - len << " bytes) back to 0" << std::endl; + + memmove(decoding_buffer.data,decoding_buffer.data+len,decoding_buffer.size - len) ; + decoding_buffer.size -= len ; + } + + return true ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index b947c1a0a..f5057e5e4 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -4,6 +4,10 @@ #include #include "interface/rsVOIP.h" +extern "C" { +#include +} + class QVideoOutputDevice ; class VideoCodec @@ -60,6 +64,7 @@ private: struct AVCodec ; struct AVCodecContext ; struct AVFrame ; +struct AVPacket ; class FFmpegVideo: public VideoCodec { @@ -78,6 +83,7 @@ private: AVCodecContext *decoding_context; AVFrame *encoding_frame_buffer ; AVFrame *decoding_frame_buffer ; + AVPacket decoding_buffer; uint64_t encoding_frame_count ; FILE *encoding_debug_file ; From 79aac23b6c49107d9a1b73fc066fbed4bf9b9331 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 23 Aug 2015 22:58:36 -0400 Subject: [PATCH 16/43] fixed video codec! --- plugins/VOIP/gui/VideoProcessor.cpp | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index cadef3048..51c97b780 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -509,10 +509,10 @@ FFmpegVideo::FFmpegVideo() if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); /* put sample parameters */ - encoding_context->bit_rate = 400000; + encoding_context->bit_rate = 200000; /* resolution must be a multiple of two */ - encoding_context->width = 176; - encoding_context->height = 144; + encoding_context->width = 352;//176; + encoding_context->height = 288;//144; /* frames per second */ encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames @@ -608,7 +608,9 @@ FFmpegVideo::~FFmpegVideo() bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; +#endif QImage input ; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) @@ -694,8 +696,8 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh voip_chunk.size = pkt.size + HEADER_SIZE; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - std::cerr << "Output : " << pkt.size << " bytes." << std::endl; #ifdef DEBUG_MPEG_VIDEO + std::cerr << "Output : " << pkt.size << " bytes." << std::endl; fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; fflush(encoding_debug_file) ; #endif @@ -716,13 +718,17 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Decoding data of size " << chunk.size << std::endl; +#endif unsigned char *buff ; if(decoding_buffer.data != NULL) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Completing buffer with size " << chunk.size - HEADER_SIZE + decoding_buffer.size << ": copying existing " << decoding_buffer.size << " bytes. Adding new " << chunk.size - HEADER_SIZE<< " bytes " << std::endl; +#endif uint32_t s = chunk.size - HEADER_SIZE + decoding_buffer.size ; unsigned char *tmp = (unsigned char*)memalign(16,s + FF_INPUT_BUFFER_PADDING_SIZE) ; @@ -738,7 +744,9 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) } else { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl; +#endif decoding_buffer.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; decoding_buffer.size = chunk.size - HEADER_SIZE ; @@ -747,7 +755,7 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) buff = decoding_buffer.data ; } - memcpy(buff,chunk.data,chunk.size - HEADER_SIZE) ; + memcpy(buff,chunk.data+HEADER_SIZE,chunk.size - HEADER_SIZE) ; int got_frame = 0 ; int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; @@ -757,13 +765,17 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << "Error decodign frame!" << std::endl; return false ; } +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Used " << len << " bytes out of " << decoding_buffer.size << ". got_frame = " << got_frame << std::endl; +#endif if(got_frame) { image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; +#endif for (int y = 0; y < decoding_frame_buffer->height; y++) for (int x = 0; x < decoding_frame_buffer->width; x++) @@ -788,7 +800,9 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) } else if(len != 0) { +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Moving remaining data (" << decoding_buffer.size - len << " bytes) back to 0" << std::endl; +#endif memmove(decoding_buffer.data,decoding_buffer.data+len,decoding_buffer.size - len) ; decoding_buffer.size -= len ; From 3f1ebca8032e7efd44389b1f2efb246c99710599 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 24 Aug 2015 21:15:33 -0400 Subject: [PATCH 17/43] added max bandwidth test preview and in/out bw estimate. Still missing the logic inside the codec --- plugins/VOIP/gui/AudioInputConfig.cpp | 9 ++++ plugins/VOIP/gui/AudioInputConfig.h | 5 ++- plugins/VOIP/gui/AudioInputConfig.ui | 35 ++++++++++++++- plugins/VOIP/gui/QVideoDevice.cpp | 32 +++---------- plugins/VOIP/gui/QVideoDevice.h | 6 +-- plugins/VOIP/gui/VOIPChatWidgetHolder.cpp | 2 +- plugins/VOIP/gui/VideoProcessor.cpp | 55 +++++++++++++++++++---- plugins/VOIP/gui/VideoProcessor.h | 21 +++++++-- 8 files changed, 120 insertions(+), 45 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 5f6aacfb8..6ef39bce5 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -120,6 +120,8 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) videoProcessor = new VideoProcessor() ; videoProcessor->setDisplayTarget(NULL) ; + videoProcessor->setMaximumBandwidth(ui.availableBW_SB->value()) ; + videoInput->setVideoProcessor(videoProcessor) ; graph_source = new voipGraphSource ; @@ -130,8 +132,15 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) graph_source->start() ; QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; + QObject::connect(ui.availableBW_SB,SIGNAL(valueChanged(double)),this,SLOT(updateAvailableBW(double))) ; } +void AudioInputConfig::updateAvailableBW(double r) +{ + std::cerr << "Setting max bandwidth to " << r << " KB/s" << std::endl; + videoProcessor->setMaximumBandwidth((uint32_t)(r*1024)) ; +} + void AudioInputConfig::togglePreview(bool b) { if(b) diff --git a/plugins/VOIP/gui/AudioInputConfig.h b/plugins/VOIP/gui/AudioInputConfig.h index e838744b4..ef53c28e9 100644 --- a/plugins/VOIP/gui/AudioInputConfig.h +++ b/plugins/VOIP/gui/AudioInputConfig.h @@ -93,8 +93,9 @@ class AudioInputConfig : public ConfigPage virtual QPixmap iconPixmap() const { return QPixmap(":/images/talking_on.svg") ; } virtual QString pageName() const { return tr("VOIP") ; } virtual QString helpText() const { return ""; } - - private slots: + +private slots: + void updateAvailableBW(double r); void loadSettings(); void emptyBuffer(); void togglePreview(bool) ; diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 7b77fb3a9..71ed06569 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -7,7 +7,7 @@ 0 0 1155 - 713 + 832 @@ -391,6 +391,39 @@ + + + + + + Available bandwidth: + + + + + + + <html><head/><body><p>Use this field to simulate the maximum bandwidth available so as to preview what the encoded video will look like with the corresponding compression rate.</p></body></html> + + + KB/s + + + 1 + + + 5.000000000000000 + + + 200.000000000000000 + + + 30.000000000000000 + + + + + diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index c3deac879..516933807 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -13,9 +13,6 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) _capture_device = NULL ; _video_processor = NULL ; _echo_output_device = NULL ; - _estimated_bw = 0 ; - _total_encoded_size = 0 ; - _last_bw_estimate_TS = time(NULL) ; } void QVideoInputDevice::stop() @@ -80,27 +77,7 @@ void QVideoInputDevice::grabFrame() if(_video_processor != NULL) { - uint32_t encoded_size ; - - _video_processor->processImage(image,0,encoded_size) ; -#ifdef DEBUG_VIDEO_OUTPUT_DEVICE - std::cerr << "Encoded size = " << encoded_size << std::endl; -#endif - _total_encoded_size += encoded_size ; - - time_t now = time(NULL) ; - - if(now > _last_bw_estimate_TS) - { - _estimated_bw = uint32_t(0.75*_estimated_bw + 0.25 * (_total_encoded_size / (float)(now - _last_bw_estimate_TS))) ; - - _total_encoded_size = 0 ; - _last_bw_estimate_TS = now ; - -#ifdef DEBUG_VIDEO_OUTPUT_DEVICE - std::cerr << "new bw estimate: " << _estimated_bw << std::endl; -#endif - } + _video_processor->processImage(image,0) ; emit networkPacketReady() ; } @@ -113,7 +90,12 @@ bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) if(_video_processor) return _video_processor->nextEncodedPacket(chunk) ; else - return false ; + return false ; +} + +uint32_t QVideoInputDevice::currentBandwidth() const +{ + return _video_processor->currentBandwidthOut() ; } QVideoInputDevice::~QVideoInputDevice() diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index d12fe5f38..20a837446 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -45,7 +45,7 @@ class QVideoInputDevice: public QObject // gets the estimated current bandwidth required to transmit the encoded data, in B/s // - uint32_t currentBandwidth() const { return _estimated_bw ; } + uint32_t currentBandwidth() const ; // control @@ -66,9 +66,5 @@ class QVideoInputDevice: public QObject QVideoOutputDevice *_echo_output_device ; std::list _out_queue ; - - uint32_t _estimated_bw ; - time_t _last_bw_estimate_TS; - uint32_t _total_encoded_size ; }; diff --git a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp index 4d1f8ff49..f8760132d 100644 --- a/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp +++ b/plugins/VOIP/gui/VOIPChatWidgetHolder.cpp @@ -365,7 +365,7 @@ void VOIPChatWidgetHolder::botMouseLeave() void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec) { - videoProcessor->setMaximumFrameRate(bytes_per_sec) ; + videoProcessor->setMaximumBandwidth(bytes_per_sec) ; } void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 51c97b780..c4934ee0c 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -32,9 +32,19 @@ VideoProcessor::VideoProcessor() //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; + + _estimated_bandwidth_in = 0 ; + _estimated_bandwidth_out = 0 ; + _target_bandwidth_out = 0 ; + + _total_encoded_size_in = 0 ; + _total_encoded_size_out = 0 ; + + _last_bw_estimate_in_TS = time(NULL) ; + _last_bw_estimate_out_TS = time(NULL) ; } -bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) +bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) { VideoCodec *codec ; @@ -52,17 +62,30 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& // std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; - encoded_size = 0 ; - if(codec) { RsVOIPDataChunk chunk ; if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) - { - encoded_size = chunk.size ; - _encoded_out_queue.push_back(chunk) ; - } + { + _encoded_out_queue.push_back(chunk) ; + + _total_encoded_size_out += chunk.size ; + } + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_out_TS) + { + _estimated_bandwidth_out = uint32_t(0.75*_estimated_bandwidth_out + 0.25 * (_total_encoded_size_out / (float)(now - _last_bw_estimate_out_TS))) ; + + _total_encoded_size_out = 0 ; + _last_bw_estimate_out_TS = now ; + +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE + std::cerr << "new bw estimate: " << _estimated_bw << std::endl; +#endif + } return true ; } @@ -134,6 +157,21 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } + _total_encoded_size_in += chunk.size ; + + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_in_TS) + { + _estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ; + + _total_encoded_size_in = 0 ; + _last_bw_estimate_in_TS = now ; + +#ifdef DEBUG_VIDEO_OUTPUT_DEVICE + std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; +#endif + } if(!codec->decodeData(chunk,img)) { std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; @@ -144,9 +182,10 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) _decoded_output_device->showFrame(img) ; } -void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec) +void VideoProcessor::setMaximumBandwidth(uint32_t bytes_per_sec) { std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; + _target_bandwidth_out = bytes_per_sec ; } ////////////////////////////////////////////////////////////////////////////////////////////////////////////// diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index f5057e5e4..230c9dc9b 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -117,7 +117,7 @@ class VideoProcessor // returns the current (measured) frame rate in bytes per second. // - uint32_t currentDecodingFrameRate() const; + uint32_t currentBandwidthIn() const { return _estimated_bandwidth_in ; } private: QVideoOutputDevice *_decoded_output_device ; @@ -130,15 +130,19 @@ class VideoProcessor public: // Takes the next image to be encoded. // - bool processImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; + bool processImage(const QImage& Image, uint32_t size_hint) ; bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; } bool nextEncodedPacket(RsVOIPDataChunk& ) ; // Used to tweak the compression ratio so that the video can stream ok. // - void setMaximumFrameRate(uint32_t bytes_per_second) ; + void setMaximumBandwidth(uint32_t bytes_per_second) ; void setInternalFrameSize(QSize) ; + // returns the current encoding frame rate in bytes per second. + // + uint32_t currentBandwidthOut() const { return _estimated_bandwidth_out ; } + protected: std::list _encoded_out_queue ; QSize _encoded_frame_size ; @@ -152,5 +156,16 @@ class VideoProcessor FFmpegVideo _mpeg_video_codec ; uint16_t _encoding_current_codec ; + + time_t _last_bw_estimate_in_TS; + time_t _last_bw_estimate_out_TS; + + uint32_t _total_encoded_size_in ; + uint32_t _total_encoded_size_out ; + + float _estimated_bandwidth_in ; + float _estimated_bandwidth_out ; + + float _target_bandwidth_out ; }; From da69db39d6a9588afa801891b3a0697b99927a2f Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 24 Aug 2015 21:42:01 -0400 Subject: [PATCH 18/43] added mutex to video processor --- plugins/VOIP/gui/VideoProcessor.cpp | 50 ++++++++++++++++++++--------- plugins/VOIP/gui/VideoProcessor.h | 4 ++- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index c4934ee0c..ef10b63a5 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -23,12 +23,13 @@ extern "C" { #include #include } -#define DEBUG_MPEG_VIDEO 1 +//#define DEBUG_MPEG_VIDEO 1 VideoProcessor::VideoProcessor() - :_encoded_frame_size(176,144) + :_encoded_frame_size(176,144) , vpMtx("VideoProcessor") { - _decoded_output_device = NULL ; + _decoded_output_device = NULL ; + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; @@ -44,6 +45,19 @@ VideoProcessor::VideoProcessor() _last_bw_estimate_out_TS = time(NULL) ; } +VideoProcessor::~VideoProcessor() +{ + // clear encoding queue + + RS_STACK_MUTEX(vpMtx) ; + + while(!_encoded_out_queue.empty()) + { + _encoded_out_queue.back().clear() ; + _encoded_out_queue.pop_back() ; + } +} + bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) { VideoCodec *codec ; @@ -68,8 +82,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) { + RS_STACK_MUTEX(vpMtx) ; _encoded_out_queue.push_back(chunk) ; - _total_encoded_size_out += chunk.size ; } @@ -77,6 +91,8 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) if(now > _last_bw_estimate_out_TS) { + RS_STACK_MUTEX(vpMtx) ; + _estimated_bandwidth_out = uint32_t(0.75*_estimated_bandwidth_out + 0.25 * (_total_encoded_size_out / (float)(now - _last_bw_estimate_out_TS))) ; _total_encoded_size_out = 0 ; @@ -98,6 +114,7 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) { + RS_STACK_MUTEX(vpMtx) ; if(_encoded_out_queue.empty()) return false ; @@ -157,21 +174,24 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) return ; } - _total_encoded_size_in += chunk.size ; - - time_t now = time(NULL) ; - - if(now > _last_bw_estimate_in_TS) { - _estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ; + RS_STACK_MUTEX(vpMtx) ; + _total_encoded_size_in += chunk.size ; - _total_encoded_size_in = 0 ; - _last_bw_estimate_in_TS = now ; + time_t now = time(NULL) ; + + if(now > _last_bw_estimate_in_TS) + { + _estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ; + + _total_encoded_size_in = 0 ; + _last_bw_estimate_in_TS = now ; #ifdef DEBUG_VIDEO_OUTPUT_DEVICE - std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; + std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; #endif - } + } + } if(!codec->decodeData(chunk,img)) { std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; @@ -794,7 +814,7 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) buff = decoding_buffer.data ; } - memcpy(buff,chunk.data+HEADER_SIZE,chunk.size - HEADER_SIZE) ; + memcpy(buff,&((unsigned char*)chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; int got_frame = 0 ; int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 230c9dc9b..a36598649 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -96,7 +96,7 @@ class VideoProcessor { public: VideoProcessor() ; - virtual ~VideoProcessor() {} + virtual ~VideoProcessor() ; enum CodecId { VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000, @@ -167,5 +167,7 @@ class VideoProcessor float _estimated_bandwidth_out ; float _target_bandwidth_out ; + + RsMutex vpMtx ; }; From 68918bf9c1a6bd7841e5d2e99762607e1d918aea Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 29 Aug 2015 21:48:58 -0400 Subject: [PATCH 19/43] added variable bitrate and attempt to control it. --- plugins/VOIP/gui/AudioInputConfig.ui | 2 +- plugins/VOIP/gui/QVideoDevice.cpp | 2 +- plugins/VOIP/gui/VideoProcessor.cpp | 52 ++++++++++++++++++++++------ plugins/VOIP/gui/VideoProcessor.h | 8 ++--- 4 files changed, 47 insertions(+), 17 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.ui b/plugins/VOIP/gui/AudioInputConfig.ui index 71ed06569..62e8c98a6 100644 --- a/plugins/VOIP/gui/AudioInputConfig.ui +++ b/plugins/VOIP/gui/AudioInputConfig.ui @@ -412,7 +412,7 @@ 1 - 5.000000000000000 + 2.000000000000000 200.000000000000000 diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 516933807..3a0cc4aa9 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -77,7 +77,7 @@ void QVideoInputDevice::grabFrame() if(_video_processor != NULL) { - _video_processor->processImage(image,0) ; + _video_processor->processImage(image) ; emit networkPacketReady() ; } diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index ef10b63a5..7da37fbbe 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -36,7 +36,7 @@ VideoProcessor::VideoProcessor() _estimated_bandwidth_in = 0 ; _estimated_bandwidth_out = 0 ; - _target_bandwidth_out = 0 ; + _target_bandwidth_out = 30*1024 ; // 30 KB/s _total_encoded_size_in = 0 ; _total_encoded_size_out = 0 ; @@ -58,7 +58,7 @@ VideoProcessor::~VideoProcessor() } } -bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) +bool VideoProcessor::processImage(const QImage& img) { VideoCodec *codec ; @@ -80,7 +80,7 @@ bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint) { RsVOIPDataChunk chunk ; - if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) && chunk.size > 0) + if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),_target_bandwidth_out,chunk) && chunk.size > 0) { RS_STACK_MUTEX(vpMtx) ; _encoded_out_queue.push_back(chunk) ; @@ -318,7 +318,7 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa } -bool WaveletVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +bool WaveletVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) { static const int WAVELET_IMG_SIZE = 128 ; static const float W_THRESHOLD = 0.005 ; // low quality @@ -568,7 +568,28 @@ FFmpegVideo::FFmpegVideo() if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); /* put sample parameters */ - encoding_context->bit_rate = 200000; + encoding_context->bit_rate = 30*1024 ; // default bitrate is 30KB/s + encoding_context->bit_rate_tolerance = encoding_context->bit_rate ; + encoding_context->rc_min_rate = 0; + encoding_context->rc_max_rate = 81920; + encoding_context->rc_buffer_size = 1024*1024; + encoding_context->flags |= CODEC_FLAG_PSNR; + //encoding_context->partitions = X264_PART_I4X4 | X264_PART_I8X8 | X264_PART_P8X8 | X264_PART_P4X4 | X264_PART_B8X8; + //encoding_context->crf = 0.0f; + //encoding_context->cqp = 26; + encoding_context->i_quant_factor = 0.769f; + encoding_context->b_quant_factor = 1.4f; + encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size); + encoding_context->rc_max_available_vbv_use = 0.9; + encoding_context->rc_min_vbv_overflow_use = 0.1; + encoding_context->time_base.num = 1; + encoding_context->time_base.den = 15;//framesPerSecond; + //encoding_context->me_method = ME_HEX; + encoding_context->qmin = 10; + encoding_context->qmax = 51; + encoding_context->max_qdiff = 4; + encoding_context->max_b_frames = 4; + /* resolution must be a multiple of two */ encoding_context->width = 352;//176; encoding_context->height = 288;//144; @@ -580,10 +601,9 @@ FFmpegVideo::FFmpegVideo() * then gop_size is ignored and the output of encoder * will always be I frame irrespective to gop_size */ - encoding_context->gop_size = 10; + encoding_context->gop_size = 250; encoding_context->max_b_frames = 1; - //context->pix_fmt = AV_PIX_FMT_RGB24; - encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; + encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; //context->pix_fmt = AV_PIX_FMT_RGB24; if (codec_id == AV_CODEC_ID_H264) av_opt_set(encoding_context->priv_data, "preset", "slow", 0); @@ -664,13 +684,22 @@ FFmpegVideo::~FFmpegVideo() free(decoding_frame_buffer); } +#define MAX_FFMPEG_ENCODING_BITRATE 81920 -bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataChunk& voip_chunk) +bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) { #ifdef DEBUG_MPEG_VIDEO std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; #endif QImage input ; + + if(target_encoding_bitrate > MAX_FFMPEG_ENCODING_BITRATE) + { + std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl; + target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ; + } + encoding_context->bit_rate = target_encoding_bitrate; + encoding_context->bit_rate_tolerance = target_encoding_bitrate; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; @@ -775,6 +804,7 @@ bool FFmpegVideo::encodeData(const QImage& image,uint32_t size_hint,RsVOIPDataCh } } + bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { #ifdef DEBUG_MPEG_VIDEO @@ -843,9 +873,9 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ; int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ; - int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; } diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index a36598649..85ecae7df 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -29,7 +29,7 @@ public: JPEGVideo() ; protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ; @@ -47,7 +47,7 @@ public: WaveletVideo() {} protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: @@ -73,7 +73,7 @@ public: ~FFmpegVideo() ; protected: - virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ; + virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; private: @@ -130,7 +130,7 @@ class VideoProcessor public: // Takes the next image to be encoded. // - bool processImage(const QImage& Image, uint32_t size_hint) ; + bool processImage(const QImage& Image) ; bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; } bool nextEncodedPacket(RsVOIPDataChunk& ) ; From e135e3441cde7b2fd18ae5735feb172c599df224 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 17:55:30 +0200 Subject: [PATCH 20/43] Create retroshare.pri for common configurations in .pro files So far it only contains the default values for installation paths --- RetroShare.pro | 7 ++----- libbitdht/src/libbitdht.pro | 2 ++ libresapi/src/libresapi.pro | 2 ++ libretroshare/src/libretroshare.pro | 7 ++----- openpgpsdk/src/openpgpsdk.pro | 2 ++ plugins/Common/retroshare_plugin.pri | 5 ++--- retroshare-gui/src/retroshare-gui.pro | 5 ++--- retroshare-nogui/src/retroshare-nogui.pro | 4 ++-- retroshare.pri | 6 ++++++ 9 files changed, 22 insertions(+), 18 deletions(-) create mode 100644 retroshare.pri diff --git a/RetroShare.pro b/RetroShare.pro index b6df6e491..aaa1bbfde 100644 --- a/RetroShare.pro +++ b/RetroShare.pro @@ -1,3 +1,5 @@ +!include("retroshare.pri"): error("Could not include file retroshare.pri") + TEMPLATE = subdirs SUBDIRS += \ @@ -32,11 +34,6 @@ plugins.file = plugins/plugins.pro plugins.depends = retroshare_gui unix { - isEmpty(PREFIX) { PREFIX = /usr } - isEmpty(INC_DIR) { INC_DIR = "$${PREFIX}/include/retroshare06" } - isEmpty(LIB_DIR) { LIB_DIR = "$${PREFIX}/lib" } - isEmpty(DATA_DIR) { DATA_DIR = "$${PREFIX}/share/RetroShare06" } - icon_files.path = "$${PREFIX}/share/icons/hicolor" icon_files.files = data/24x24 icon_files.files += data/48x48 diff --git a/libbitdht/src/libbitdht.pro b/libbitdht/src/libbitdht.pro index 6f705fa7d..ed576a700 100644 --- a/libbitdht/src/libbitdht.pro +++ b/libbitdht/src/libbitdht.pro @@ -1,3 +1,5 @@ +!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") + TEMPLATE = lib CONFIG += staticlib CONFIG -= qt diff --git a/libresapi/src/libresapi.pro b/libresapi/src/libresapi.pro index a7a081701..cc12e1503 100644 --- a/libresapi/src/libresapi.pro +++ b/libresapi/src/libresapi.pro @@ -1,3 +1,5 @@ +!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") + TEMPLATE = lib CONFIG += staticlib CONFIG -= qt diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index dca32bcd1..ba29fa605 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -1,3 +1,5 @@ +!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") + TEMPLATE = lib CONFIG += staticlib bitdht CONFIG -= qt @@ -168,11 +170,6 @@ linux-* { } unix { - isEmpty(PREFIX) { PREFIX = /usr } - isEmpty(INC_DIR) { INC_DIR = "$${PREFIX}/include/retroshare06" } - isEmpty(LIB_DIR) { LIB_DIR = "$${PREFIX}/lib" } - isEmpty(DATA_DIR) { DATA_DIR = "$${PREFIX}/share/RetroShare06" } - DEFINES *= LIB_DIR=\"\\\"$${LIB_DIR}\\\"\" DEFINES *= DATA_DIR=\"\\\"$${DATA_DIR}\\\"\" diff --git a/openpgpsdk/src/openpgpsdk.pro b/openpgpsdk/src/openpgpsdk.pro index c0f741857..786ee5819 100644 --- a/openpgpsdk/src/openpgpsdk.pro +++ b/openpgpsdk/src/openpgpsdk.pro @@ -1,3 +1,5 @@ +!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") + TEMPLATE = lib win32 { CONFIG += staticlib diff --git a/plugins/Common/retroshare_plugin.pri b/plugins/Common/retroshare_plugin.pri index e77626dc6..d20c1baa4 100644 --- a/plugins/Common/retroshare_plugin.pri +++ b/plugins/Common/retroshare_plugin.pri @@ -1,3 +1,5 @@ +!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") + TEMPLATE = lib CONFIG *= plugin @@ -5,9 +7,6 @@ DEPENDPATH += ../../libretroshare/src/ ../../retroshare-gui/src/ INCLUDEPATH += ../../libretroshare/src/ ../../retroshare-gui/src/ unix { - isEmpty(PREFIX) { PREFIX = /usr } - isEmpty(LIB_DIR) { LIB_DIR = "$${PREFIX}/lib" } - target.path = "$${LIB_DIR}/retroshare/extensions6" INSTALLS += target } diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 6e61a81e2..6632c871f 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -1,3 +1,5 @@ +!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") + QT += network xml script CONFIG += qt gui uic qrc resources idle bitdht @@ -110,9 +112,6 @@ linux-* { } unix { - isEmpty(PREFIX) { PREFIX = /usr } - isEmpty(DATA_DIR) { DATA_DIR = "$${PREFIX}/share/RetroShare06" } - target.path = "$${PREFIX}/bin" INSTALLS += target diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index 9d590efba..52dde1f02 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -1,3 +1,5 @@ +!include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") + TEMPLATE = app TARGET = RetroShare06-nogui CONFIG += bitdht @@ -56,8 +58,6 @@ linux-* { } unix { - isEmpty(PREFIX) { PREFIX = /usr } - target.path = "$${PREFIX}/bin" INSTALLS += target } diff --git a/retroshare.pri b/retroshare.pri new file mode 100644 index 000000000..fb20b1984 --- /dev/null +++ b/retroshare.pri @@ -0,0 +1,6 @@ +unix { + isEmpty(PREFIX) { PREFIX = "/usr" } + isEmpty(INC_DIR) { INC_DIR = "$${PREFIX}/include/retroshare06" } + isEmpty(LIB_DIR) { LIB_DIR = "$${PREFIX}/lib" } + isEmpty(DATA_DIR) { DATA_DIR = "$${PREFIX}/share/RetroShare06" } +} From b66f466c7a6bd2acb2fc7cf6ac5df868a463021a Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 18:01:18 +0200 Subject: [PATCH 21/43] Move all INSTALLS to the subproject they belong to Allows installation of for example only retroshare-nogui, without installing files only relevant for gui $ qmake $ make retroshare-nogui $ make retroshare-nogui-install_subtargets --- RetroShare.pro | 32 +++------------------------ libbitdht/src/libbitdht.pro | 7 ++++++ libresapi/src/libresapi.pro | 10 +++++++++ retroshare-gui/src/retroshare-gui.pro | 16 ++++++++++++++ 4 files changed, 36 insertions(+), 29 deletions(-) diff --git a/RetroShare.pro b/RetroShare.pro index aaa1bbfde..4a1be6b49 100644 --- a/RetroShare.pro +++ b/RetroShare.pro @@ -26,38 +26,12 @@ pegmarkdown.file = supportlibs/pegmarkdown/pegmarkdown.pro retroshare_gui.file = retroshare-gui/src/retroshare-gui.pro retroshare_gui.depends = libretroshare libresapi pegmarkdown +retroshare_gui.target = retroshare-gui retroshare_nogui.file = retroshare-nogui/src/retroshare-nogui.pro retroshare_nogui.depends = libretroshare libresapi +retroshare_nogui.target = retroshare-nogui plugins.file = plugins/plugins.pro plugins.depends = retroshare_gui - -unix { - icon_files.path = "$${PREFIX}/share/icons/hicolor" - icon_files.files = data/24x24 - icon_files.files += data/48x48 - icon_files.files += data/64x64 - icon_files.files += data/128x128 - INSTALLS += icon_files - - desktop_files.path = "$${PREFIX}/share/applications" - desktop_files.files = data/retroshare06.desktop - INSTALLS += desktop_files - - pixmap_files.path = "$${PREFIX}/share/pixmaps" - pixmap_files.files = data/retroshare06.xpm - INSTALLS += pixmap_files - - data_files.path = "$${DATA_DIR}" - data_files.files = libbitdht/src/bitdht/bdboot.txt - INSTALLS += data_files - - webui_files.path = "$${DATA_DIR}/webui" - webui_files.files = libresapi/src/webfiles/* - INSTALLS += webui_files - - webui_img_files.path = "$${DATA_DIR}/webui/img" - webui_img_files.files = retroshare-gui/src/gui/images/logo/logo_splash.png - INSTALLS += webui_img_files -} +plugins.target = plugins diff --git a/libbitdht/src/libbitdht.pro b/libbitdht/src/libbitdht.pro index ed576a700..7d6dca1d6 100644 --- a/libbitdht/src/libbitdht.pro +++ b/libbitdht/src/libbitdht.pro @@ -39,6 +39,13 @@ linux-g++-64 { OBJECTS_DIR = temp/linux-g++-64/obj } +unix { + data_files.path = "$${DATA_DIR}" + data_files.files = bitdht/bdboot.txt + INSTALLS += data_files +} + + #################### Cross compilation for windows under Linux #################### win32-x-g++ { diff --git a/libresapi/src/libresapi.pro b/libresapi/src/libresapi.pro index cc12e1503..a45255e45 100644 --- a/libresapi/src/libresapi.pro +++ b/libresapi/src/libresapi.pro @@ -10,6 +10,16 @@ CONFIG += libmicrohttpd INCLUDEPATH += ../../libretroshare/src +unix { + webui_files.path = "$${DATA_DIR}/webui" + webui_files.files = webfiles/* + INSTALLS += webui_files + + webui_img_files.path = "$${DATA_DIR}/webui/img" + webui_img_files.files = ../../retroshare-gui/src/gui/images/logo/logo_splash.png + INSTALLS += webui_img_files +} + win32{ DEFINES *= WINDOWS_SYS INCLUDEPATH += $$PWD/../../../libs/include diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 6632c871f..57a8a88d8 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -122,6 +122,22 @@ unix { style_files.path="$${DATA_DIR}/stylesheets" style_files.files=gui/qss/chat/Bubble gui/qss/chat/Bubble_Compact INSTALLS += style_files + + icon_files.path = "$${PREFIX}/share/icons/hicolor" + icon_files.files = ../../data/24x24 + icon_files.files += ../../data/48x48 + icon_files.files += ../../data/64x64 + icon_files.files += ../../data/128x128 + INSTALLS += icon_files + + desktop_files.path = "$${PREFIX}/share/applications" + desktop_files.files = ../../data/retroshare06.desktop + INSTALLS += desktop_files + + pixmap_files.path = "$${PREFIX}/share/pixmaps" + pixmap_files.files = ../../data/retroshare06.xpm + INSTALLS += pixmap_files + } linux-g++ { From b9c174ba8f1f156d788191a75a8b8a9d4c273ea7 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 18:01:55 +0200 Subject: [PATCH 22/43] Add BIN_DIR variable to allow packagers to override it --- retroshare-gui/src/retroshare-gui.pro | 2 +- retroshare-nogui/src/retroshare-nogui.pro | 2 +- retroshare.pri | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 57a8a88d8..ec967c51a 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -112,7 +112,7 @@ linux-* { } unix { - target.path = "$${PREFIX}/bin" + target.path = "$${BIN_DIR}" INSTALLS += target data_files.path="$${DATA_DIR}/" diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index 52dde1f02..9b8136942 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -58,7 +58,7 @@ linux-* { } unix { - target.path = "$${PREFIX}/bin" + target.path = "$${BIN_DIR}" INSTALLS += target } diff --git a/retroshare.pri b/retroshare.pri index fb20b1984..b85b40cbc 100644 --- a/retroshare.pri +++ b/retroshare.pri @@ -1,5 +1,6 @@ unix { isEmpty(PREFIX) { PREFIX = "/usr" } + isEmpty(BIN_DIR) { BIN_DIR = "$${PREFIX}/bin" } isEmpty(INC_DIR) { INC_DIR = "$${PREFIX}/include/retroshare06" } isEmpty(LIB_DIR) { LIB_DIR = "$${PREFIX}/lib" } isEmpty(DATA_DIR) { DATA_DIR = "$${PREFIX}/share/RetroShare06" } From dca295143aee6c8e39f12b6de3f5d75eb49d268f Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 19:20:52 +0200 Subject: [PATCH 23/43] Use CONFIG+=create_prl for libretroshare Allows to specify all LIBS in libretroshare.pro so they don't also have to be specified in gui/nogui.pro --- libretroshare/src/libretroshare.pro | 1 + retroshare-gui/src/retroshare-gui.pro | 1 + retroshare-nogui/src/retroshare-nogui.pro | 1 + 3 files changed, 3 insertions(+) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index ba29fa605..433c49b73 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -2,6 +2,7 @@ TEMPLATE = lib CONFIG += staticlib bitdht +CONFIG += create_prl CONFIG -= qt TARGET = retroshare diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index ec967c51a..00fd3a5e4 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -2,6 +2,7 @@ QT += network xml script CONFIG += qt gui uic qrc resources idle bitdht +CONFIG += link_prl # Plz never commit the .pro with these flags enabled. # Use this flag when developping new features only. diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index 9b8136942..fe5c7d6a6 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -8,6 +8,7 @@ CONFIG += bitdht # webinterface, requires libmicrohttpd CONFIG += webui CONFIG -= qt xml gui +CONFIG += link_prl # if you are linking against the libretroshare with gxs. # this option links against the required sqlite library. From 3d9e3e8b7cb06fd1e7c0110e5e423cbec1c019fa Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 31 Aug 2015 22:41:38 -0400 Subject: [PATCH 24/43] attempt to setup variable bitrate. Improved codec params --- plugins/VOIP/gui/VideoProcessor.cpp | 61 +++++++++++++++++------------ 1 file changed, 37 insertions(+), 24 deletions(-) diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 7da37fbbe..26fa74a53 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -26,7 +26,7 @@ extern "C" { //#define DEBUG_MPEG_VIDEO 1 VideoProcessor::VideoProcessor() - :_encoded_frame_size(176,144) , vpMtx("VideoProcessor") + :_encoded_frame_size(640,480) , vpMtx("VideoProcessor") { _decoded_output_device = NULL ; @@ -554,7 +554,8 @@ FFmpegVideo::FFmpegVideo() encoding_context = NULL ; //AVCodecID codec_id = AV_CODEC_ID_H264 ; - AVCodecID codec_id = AV_CODEC_ID_MPEG1VIDEO; + //AVCodecID codec_id = AV_CODEC_ID_MPEG2VIDEO; + AVCodecID codec_id = AV_CODEC_ID_MPEG4; uint8_t endcode[] = { 0, 0, 1, 0xb7 }; @@ -568,31 +569,42 @@ FFmpegVideo::FFmpegVideo() if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); /* put sample parameters */ - encoding_context->bit_rate = 30*1024 ; // default bitrate is 30KB/s + encoding_context->bit_rate = 10*1024 ; // default bitrate is 30KB/s encoding_context->bit_rate_tolerance = encoding_context->bit_rate ; + +#ifdef USE_VARIABLE_BITRATE encoding_context->rc_min_rate = 0; - encoding_context->rc_max_rate = 81920; - encoding_context->rc_buffer_size = 1024*1024; + encoding_context->rc_max_rate = 10*1024;//encoding_context->bit_rate; + encoding_context->rc_buffer_size = 10*1024*1024; + encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size); + encoding_context->rc_max_available_vbv_use = 1.0; + encoding_context->rc_min_vbv_overflow_use = 0.0; +#else + encoding_context->rc_min_rate = 0; + encoding_context->rc_max_rate = 0; + encoding_context->rc_buffer_size = 0; +#endif encoding_context->flags |= CODEC_FLAG_PSNR; + encoding_context->flags |= CODEC_FLAG_TRUNCATED; + encoding_context->flags |= CODEC_CAP_PARAM_CHANGE; + encoding_context->i_quant_factor = 0.769f; + encoding_context->b_quant_factor = 1.4f; + encoding_context->time_base.num = 1; + encoding_context->time_base.den = 15;//framesPerSecond; + encoding_context->qmin = 1; + encoding_context->qmax = 51; + encoding_context->max_qdiff = 4; + + //encoding_context->me_method = ME_HEX; + //encoding_context->max_b_frames = 4; + //encoding_context->flags |= CODEC_FLAG_LOW_DELAY; // MPEG2 only //encoding_context->partitions = X264_PART_I4X4 | X264_PART_I8X8 | X264_PART_P8X8 | X264_PART_P4X4 | X264_PART_B8X8; //encoding_context->crf = 0.0f; //encoding_context->cqp = 26; - encoding_context->i_quant_factor = 0.769f; - encoding_context->b_quant_factor = 1.4f; - encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size); - encoding_context->rc_max_available_vbv_use = 0.9; - encoding_context->rc_min_vbv_overflow_use = 0.1; - encoding_context->time_base.num = 1; - encoding_context->time_base.den = 15;//framesPerSecond; - //encoding_context->me_method = ME_HEX; - encoding_context->qmin = 10; - encoding_context->qmax = 51; - encoding_context->max_qdiff = 4; - encoding_context->max_b_frames = 4; - + /* resolution must be a multiple of two */ - encoding_context->width = 352;//176; - encoding_context->height = 288;//144; + encoding_context->width = 640;//176; + encoding_context->height = 480;//144; /* frames per second */ encoding_context->time_base = (AVRational){1,25}; /* emit one intra frame every ten frames @@ -601,8 +613,8 @@ FFmpegVideo::FFmpegVideo() * then gop_size is ignored and the output of encoder * will always be I frame irrespective to gop_size */ - encoding_context->gop_size = 250; - encoding_context->max_b_frames = 1; + encoding_context->gop_size = 100; + //encoding_context->max_b_frames = 1; encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; //context->pix_fmt = AV_PIX_FMT_RGB24; if (codec_id == AV_CODEC_ID_H264) @@ -698,8 +710,9 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl; target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ; } - encoding_context->bit_rate = target_encoding_bitrate; - encoding_context->bit_rate_tolerance = target_encoding_bitrate; + //encoding_context->bit_rate = target_encoding_bitrate; + encoding_context->rc_max_rate = target_encoding_bitrate; + //encoding_context->bit_rate_tolerance = target_encoding_bitrate; if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height) input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ; From c5cd701d4c516fa6e51887b9a83f2aaf1a35db2e Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 31 Aug 2015 22:44:34 -0400 Subject: [PATCH 25/43] removed experimental wavelet codec --- plugins/VOIP/gui/DaubechyWavelets.h | 267 ---------------------------- plugins/VOIP/gui/VideoProcessor.cpp | 234 ------------------------ plugins/VOIP/gui/VideoProcessor.h | 21 --- 3 files changed, 522 deletions(-) delete mode 100644 plugins/VOIP/gui/DaubechyWavelets.h diff --git a/plugins/VOIP/gui/DaubechyWavelets.h b/plugins/VOIP/gui/DaubechyWavelets.h deleted file mode 100644 index c69f025b0..000000000 --- a/plugins/VOIP/gui/DaubechyWavelets.h +++ /dev/null @@ -1,267 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include - -#ifdef USE_SSE_INSTRUCTIONS -#include -#endif - -template class DaubechyWavelets -{ - public: - typedef enum { DWT_DAUB02=2, DWT_DAUB04=4, DWT_DAUB12=12, DWT_DAUB20=20 } WaveletType ; - typedef enum { DWT_FORWARD=1, DWT_BACKWARD=0 } TransformType ; - - static void DWT2D(FLOAT *data,unsigned long int W,unsigned long int H,WaveletType type,TransformType tr) - { - unsigned long int nn[2] = {W,H} ; - wtn(&data[-1], &nn[-1],2, tr, waveletFilter(type), pwt) ; - } - static void DWT1D(FLOAT *data,unsigned long int W,WaveletType type,TransformType tr) - { - unsigned long int nn[1] = {W} ; - wtn(&data[-1], &nn[-1],1, tr, waveletFilter(type), pwt) ; - } - - - private: - class wavefilt - { - public: - wavefilt(int n) - { - int k; - FLOAT sig = -1.0; - static const FLOAT c2[5]={ 0.0, sqrt(2.0)/2.0, sqrt(2.0)/2.0, 0.0, 0.0 }; - - static const FLOAT c4[5]={ 0.0, 0.4829629131445341, 0.8365163037378079, 0.2241438680420134,-0.1294095225512604 }; - - static const FLOAT c12[13]={0.0,0.111540743350, 0.494623890398, 0.751133908021, - 0.315250351709,-0.226264693965,-0.129766867567, - 0.097501605587, 0.027522865530,-0.031582039318, - 0.000553842201, 0.004777257511,-0.001077301085}; - - static const FLOAT c20[21]={0.0,0.026670057901, 0.188176800078, 0.527201188932, - 0.688459039454, 0.281172343661,-0.249846424327, - -0.195946274377, 0.127369340336, 0.093057364604, - -0.071394147166,-0.029457536822, 0.033212674059, - 0.003606553567,-0.010733175483, 0.001395351747, - 0.001992405295,-0.000685856695,-0.000116466855, - 0.000093588670,-0.000013264203 }; - - ncof= (n==2)?4:n; - const FLOAT *tmpcc ; - cc.resize(ncof+1) ; - cr.resize(ncof+1) ; - - if (n == 2) - { - tmpcc=c2; - cc[1] = tmpcc[1] ; - cc[2] = tmpcc[2] ; - cc[3] = 0.0f ; - cc[4] = 0.0f ; - cr[1] = tmpcc[1] ; - cr[2] =-tmpcc[2] ; - cr[3] = 0.0f ; - cr[4] = 0.0f ; - - ioff = joff = -1 ; - } - else - { - if (n == 4) - tmpcc=c4; - else if (n == 12) - tmpcc=c12; - else if (n == 20) - tmpcc=c20; - else - throw std::runtime_error("unimplemented value n in pwtset"); - - for (k=1;k<=n;k++) - { - cc[k] = tmpcc[k] ; - cr[ncof+1-k]=sig*tmpcc[k]; - sig = -sig; - } - ioff = joff = -(n >> 1); - } - } - - ~wavefilt() {} - - int ncof,ioff,joff; - std::vector cc; - std::vector cr; - } ; - - static const wavefilt& waveletFilter(WaveletType type) - { - static wavefilt *daub02filt = NULL ; - static wavefilt *daub04filt = NULL ; - static wavefilt *daub12filt = NULL ; - static wavefilt *daub20filt = NULL ; - - switch(type) - { - case DWT_DAUB02: if(daub02filt == NULL) - daub02filt = new wavefilt(2) ; - return *daub02filt ; - - case DWT_DAUB04: if(daub04filt == NULL) - daub04filt = new wavefilt(4) ; - return *daub04filt ; - - case DWT_DAUB12: if(daub12filt == NULL) - daub12filt = new wavefilt(12) ; - return *daub12filt ; - - case DWT_DAUB20: if(daub20filt == NULL) - daub20filt = new wavefilt(20) ; - return *daub20filt ; - - default: - throw std::runtime_error("Unknown wavelet type.") ; - } - } - - static void pwt(FLOAT a[], unsigned long n, int isign,const wavefilt& wfilt) - { -/********************** BEGIN SIGNED PART *************************/ -/** md5sum = 2b9e1e38ac690f50806873cdb4a061ea **/ -/** Validation date = 08/10/10 **/ -/******************************************************************/ - unsigned long i,ii,ni,nj ; - - if (n < 4) - return; - - FLOAT *wksp=new FLOAT[n+1];//vector(1,n); - FLOAT ai,ai1 ; - unsigned long int nmod=wfilt.ncof*n; - unsigned long int n1=n-1; - unsigned long int nh=n >> 1; - - memset(wksp,0,(n+1)*sizeof(FLOAT)) ; - - if (isign == DWT_FORWARD) - for (ii=1,i=1;i<=n;i+=2,ii++) - { - ni=i+nmod+wfilt.ioff; - nj=i+nmod+wfilt.joff; - -#ifdef USE_SSE_INSTRUCTIONS -#warning Using SSE2 Instruction set for wavelet internal loops - for (int k=1;k<=wfilt.ncof;k+=4) - { - int jf=ni+k; - int jr=nj+k; - - sse_block w1(wfilt.cc[k],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) ; - sse_block w2(wfilt.cr[k],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) ; - - sse_block a1( a[1+((jf+0)&n1)], a[1+((jf+1)&n1)], a[1+((jf+2)&n1)], a[1+((jf+3)&n1)]) ; - sse_block a2( a[1+((jr+0)&n1)], a[1+((jr+1)&n1)], a[1+((jr+2)&n1)], a[1+((jr+3)&n1)]) ; - - sse_block wk1( w1*a1 ) ; - sse_block wk2( w2*a2 ) ; - - wksp[ii ] += wk1.sum() ; - wksp[ii+nh] += wk2.sum() ; - } -#else - for (int k=1;k<=wfilt.ncof;k++) - { - int jf=n1 & (ni+k); - int jr=n1 & (nj+k); - wksp[ii] += wfilt.cc[k]*a[jf+1]; - wksp[ii+nh] += wfilt.cr[k]*a[jr+1]; - } -#endif - } - else - for (ii=1,i=1;i<=n;i+=2,ii++) - { - ai=a[ii]; - ai1=a[ii+nh]; - ni=i+nmod+wfilt.ioff; - nj=i+nmod+wfilt.joff; - -#ifdef USE_SSE_INSTRUCTIONS - sse_block ai_sse( ai,ai,ai,ai ) ; - sse_block ai1_sse( ai1,ai1,ai1,ai1 ) ; - - for (int k=1;k<=wfilt.ncof;k+=4) - { - int jf=ni+k ; - int jr=nj+k ; // in fact we have jf==jr, so the code is simpler. - - sse_block w1(wksp[1+((jf+0) & n1)],wksp[1+((jf+1) & n1)],wksp[1+((jf+2) & n1)],wksp[1+((jf+3) & n1)]) ; - - w1 += sse_block(wfilt.cc[k+0],wfilt.cc[k+1],wfilt.cc[k+2],wfilt.cc[k+3]) * ai_sse ; - w1 += sse_block(wfilt.cr[k+0],wfilt.cr[k+1],wfilt.cr[k+2],wfilt.cr[k+3]) * ai1_sse ; - - wksp[1+((jr+0) & n1)] = w1[0] ; - wksp[1+((jr+1) & n1)] = w1[1] ; - wksp[1+((jr+2) & n1)] = w1[2] ; - wksp[1+((jr+3) & n1)] = w1[3] ; - } -#else - for (int k=1;k<=wfilt.ncof;++k) - { - wksp[(n1 & (ni+k))+1] += wfilt.cc[k]*ai; - wksp[(n1 & (nj+k))+1] += wfilt.cr[k]*ai1; - } -#endif - } - - for (uint j=1;j<=n;j++) - a[j]=wksp[j]; - - delete[] wksp ;//free_vector(wksp,1,n); -/********************** END SIGNED PART *************************/ - } - - static void wtn(FLOAT a[], unsigned long nn[], int ndim, int isign, const wavefilt& w,void (*wtstep)(FLOAT [], unsigned long, int,const wavefilt&)) - { - unsigned long i1,i2,i3,k,n,nnew,nprev=1,nt,ntot=1; - int idim; - FLOAT *wksp; - - for (idim=1;idim<=ndim;idim++) - ntot *= nn[idim]; - - wksp=new FLOAT[ntot+1] ; //vector(1,ntot); - - for (idim=1;idim<=ndim;idim++) - { - n=nn[idim]; - nnew=n*nprev; - - if (n > 4) - for (i2=0;i2=4;nt >>= 1) - (*wtstep)(wksp,nt,isign,w); - else - for(nt=4;nt<=n;nt <<= 1) - (*wtstep)(wksp,nt,isign,w); - - for (i3=i1+i2,k=1;k<=n;k++,i3+=nprev) a[i3]=wksp[k]; - } - - nprev=nnew; - } - delete[] wksp ;//free_vector(wksp,1,ntot); - } -}; - diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 26fa74a53..a0084ea18 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -8,7 +8,6 @@ #include "VideoProcessor.h" #include "QVideoDevice.h" -#include "DaubechyWavelets.h" #include @@ -31,7 +30,6 @@ VideoProcessor::VideoProcessor() _decoded_output_device = NULL ; //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; - //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; _estimated_bandwidth_in = 0 ; @@ -66,8 +64,6 @@ bool VideoProcessor::processImage(const QImage& img) { case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; break ; - case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; - break ; case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; break ; default: @@ -159,8 +155,6 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) { case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ; break ; - case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ; - break ; case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ; break ; default: @@ -317,234 +311,6 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa return true ; } - -bool WaveletVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) -{ - static const int WAVELET_IMG_SIZE = 128 ; - static const float W_THRESHOLD = 0.005 ; // low quality - //static const float W_THRESHOLD = 0.0001; // high quality - //static const float W_THRESHOLD = 0.0005; // medium quality - - static const int W2 = WAVELET_IMG_SIZE ; - static const int H2 = WAVELET_IMG_SIZE ; - - assert(image.width() == W2) ; - assert(image.height() == H2) ; - - float *temp = new float[W2*H2] ; - - std::cerr << " codec type: wavelets." << std::endl; - - // We should perform some interpolation here ;-) - // - for(int i=0;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_FORWARD) ; - - // Now estimate the max energy in the W coefs, and only keep the largest. - - float mx = 0.0f ; - for(int i=0;i compressed_values ; - compressed_values.reserve(W2*H2) ; - - for(int i=0;i= W_THRESHOLD*mx) // This needs to be improved. Wavelets do not all have the same visual impact. - { - // add one value, using 16 bits for coordinates and 16 bits for the value. - - compressed_values.push_back((uint16_t)i) ; - compressed_values.push_back(quantize_16b(temp[i],mx)) ; - - //float f2 = from_quantized_16b(quantize_16b(temp[i],mx),mx) ; - - //if(fabs(f2 - temp[i]) >= 0.01*(fabs(temp[i])+fabs(f2))) - //std::cerr << " before: " << temp[i] << ", quantised=" << quantize_16b(temp[i],mx)<< ", after: " << f2 << std::endl; - } - delete[] temp ; - - // Serialise all values into a memory buffer. This needs to be taken care of because of endian issues. - - int compressed_size = 4 + compressed_values.size()*2 ; - - std::cerr << " threshold : " << W_THRESHOLD << std::endl; - std::cerr << " values kept: " << compressed_values.size()/2 << std::endl; - std::cerr << " compression: " << compressed_size/float(W2*H2*3)*100 << " %" << std::endl; - - voip_chunk.data = malloc(HEADER_SIZE + compressed_size) ; - - // build header - uint32_t flags = 0 ; - - ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO & 0xff ; - ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO >> 8) & 0xff ; - ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ; - ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ; - - unsigned char *compressed_mem = &((unsigned char *)voip_chunk.data)[HEADER_SIZE] ; - serialise_ufloat(compressed_mem,mx) ; - - for(uint32_t i=0;i> 8 ; - } - - voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; - voip_chunk.size = HEADER_SIZE + compressed_size ; - - return true ; -} - -bool WaveletVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) -{ - static const int WAVELET_IMG_SIZE = 128 ; - - static const int W2 = WAVELET_IMG_SIZE ; - static const int H2 = WAVELET_IMG_SIZE ; - - float *temp = new float[W2*H2] ; - - const unsigned char *compressed_mem = &static_cast(chunk.data)[HEADER_SIZE] ; - int compressed_size = chunk.size - HEADER_SIZE; - - memset(temp,0,W2*H2*sizeof(float)) ; - float M = deserialise_ufloat(compressed_mem); - -#ifdef VOIP_CODEC_DEBUG - std::cerr << " codec type: wavelets." << std::endl; - std::cerr << " max coef: " << M << std::endl; -#endif - - for(int i=4;i::DWT2D(temp,W2,H2,DaubechyWavelets::DWT_DAUB12,DaubechyWavelets::DWT_BACKWARD) ; - -#ifdef VOIP_CODEC_DEBUG - std::cerr << " resizing image to: " << w << "x" << h << std::endl; -#endif - - image = QImage(W2,H2,QImage::Format_RGB32) ; - - int indx = 0 ; - - for(int j=0;j1023), and p is coded on 6 bits (0->63). - // Packing [mp] into a 16bit uint16_t. M is the maximum coefficient over the quantization - // process. - // - // So this represents numbers from M * 1 * 2^{-73} to M - // - // All calculatoins are performed on x/M*2^10 - // - static const float LOG2 = log(2.0f) ; - - int m,p ; - - if(fabs(x) < 1e-8*M) - { - m = 0 ; - p = 0 ; - } - else - { - float log2f = log(fabsf(x)/M)/LOG2 ; - int mexp = (int)floor(MANTISSE_BITS - log2f) ; - - m = (int)floor(pow(2.0f,mexp+log2f)) ; - p = mexp ; - - if(p > (1<> EXPONENT_BITS ; - - if(p > 10) - return M * m / 1024.0f / (float)(1 << (p-10)) ; - else - return M * m / (float)(1 << p) ; -} - -void WaveletVideo::serialise_ufloat(unsigned char *mem, float f) -{ - if(f < 0.0f) - { - std::cerr << "(EE) Cannot serialise invalid negative float value " << f << " in " << __PRETTY_FUNCTION__ << std::endl; - return ; - } - // This serialisation is quite accurate. The max relative error is approx. - // 0.01% and most of the time less than 1e-05% The error is well distributed - // over numbers also. - // - uint32_t n = (f < 1e-7)?(~(uint32_t)0): ((uint32_t)( (1.0f/(1.0f+f) * (~(uint32_t)0)))) ; - - mem[0] = n & 0xff ; n >>= 8 ; - mem[1] = n & 0xff ; n >>= 8 ; - mem[2] = n & 0xff ; n >>= 8 ; - mem[3] = n & 0xff ; -} -float WaveletVideo::deserialise_ufloat(const unsigned char *mem) -{ - uint32_t n = mem[3] ; - n = (n << 8) + mem[2] ; - n = (n << 8) + mem[1] ; - n = (n << 8) + mem[0] ; - - return 1.0f/ ( n/(float)(~(uint32_t)0)) - 1.0f ; -} - FFmpegVideo::FFmpegVideo() { // Encoding diff --git a/plugins/VOIP/gui/VideoProcessor.h b/plugins/VOIP/gui/VideoProcessor.h index 85ecae7df..a3869dc5e 100644 --- a/plugins/VOIP/gui/VideoProcessor.h +++ b/plugins/VOIP/gui/VideoProcessor.h @@ -41,26 +41,6 @@ private: uint32_t _encoded_ref_frame_count ; }; -class WaveletVideo: public VideoCodec -{ -public: - WaveletVideo() {} - -protected: - virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ; - virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ; -private: - - static const int MANTISSE_BITS = 9 ; - static const int EXPONENT_BITS = 6 ; - - static void serialise_ufloat(unsigned char *mem, float f); - static float deserialise_ufloat(const unsigned char *mem); - - static float from_quantized_16b(uint16_t n, float M); - static uint16_t quantize_16b(float x, float M); -}; - struct AVCodec ; struct AVCodecContext ; struct AVFrame ; @@ -152,7 +132,6 @@ class VideoProcessor // ===================================================================================== JPEGVideo _jpeg_video_codec ; - WaveletVideo _ddwt_video_codec ; FFmpegVideo _mpeg_video_codec ; uint16_t _encoding_current_codec ; From b1572c29243203f56a1d66b50af14abd3345c1cb Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 31 Aug 2015 22:50:20 -0400 Subject: [PATCH 26/43] added missing deps for avcodec --- build_scripts/Debian+Ubuntu/control.precise | 2 +- build_scripts/Debian+Ubuntu/control.squeeze_bubba3 | 2 +- build_scripts/Debian+Ubuntu/control.ubuntu_lucid | 2 +- build_scripts/Debian+Ubuntu/debian/control | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/build_scripts/Debian+Ubuntu/control.precise b/build_scripts/Debian+Ubuntu/control.precise index b008cf5fa..70d77ea50 100644 --- a/build_scripts/Debian+Ubuntu/control.precise +++ b/build_scripts/Debian+Ubuntu/control.precise @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 b/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 index 244bdb2a0..c72507d0f 100644 --- a/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 +++ b/build_scripts/Debian+Ubuntu/control.squeeze_bubba3 @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/control.ubuntu_lucid b/build_scripts/Debian+Ubuntu/control.ubuntu_lucid index 1188ca8a0..ae12f4247 100644 --- a/build_scripts/Debian+Ubuntu/control.ubuntu_lucid +++ b/build_scripts/Debian+Ubuntu/control.ubuntu_lucid @@ -2,7 +2,7 @@ Source: retroshare Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev, libavcodec-dev Standards-Version: 3.9.1 Homepage: http://retroshare.sourceforge.net diff --git a/build_scripts/Debian+Ubuntu/debian/control b/build_scripts/Debian+Ubuntu/debian/control index 357150da5..b251bb923 100644 --- a/build_scripts/Debian+Ubuntu/debian/control +++ b/build_scripts/Debian+Ubuntu/debian/control @@ -2,7 +2,7 @@ Source: retroshare06 Section: devel Priority: standard Maintainer: Cyril Soler -Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev +Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev Standards-Version: 3.9.3 Homepage: http://retroshare.sourceforge.net From 846f7d02044f5802412349f2f8d09685c01c5ec1 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 19:25:10 +0200 Subject: [PATCH 27/43] Make usage of sqlcipher explicit at compile time Either the build fails if it is not available, or the user specifies CONFIG+=NO_SQLCIPHER to build without it, even if it exists in the system. --- libretroshare/src/libretroshare.pro | 23 +++++++++++++++-------- retroshare-gui/src/retroshare-gui.pro | 21 --------------------- retroshare-nogui/src/retroshare-nogui.pro | 21 --------------------- 3 files changed, 15 insertions(+), 50 deletions(-) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 433c49b73..6b731a608 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -138,15 +138,22 @@ linux-* { DEPENDPATH += . $${SSL_DIR} $${UPNP_DIR} INCLUDEPATH += . $${SSL_DIR} $${UPNP_DIR} - SQLCIPHER_OK = $$system(pkg-config --exists sqlcipher && echo yes) - isEmpty(SQLCIPHER_OK) { -# We need a explicit path here, to force using the home version of sqlite3 that really encrypts the database. - !exists(../../../lib/sqlcipher/.libs/libsqlcipher.a) { - message(libsqlcipher.a not found. Compilation will not use SQLCIPER. Database will be unencrypted.) - DEFINES *= NO_SQLCIPHER + contains(CONFIG, NO_SQLCIPHER) { + DEFINES *= NO_SQLCIPHER + LIBS *= -lsqlite3 + } else { + SQLCIPHER_OK = $$system(pkg-config --exists sqlcipher && echo yes) + isEmpty(SQLCIPHER_OK) { + # We need a explicit path here, to force using the home version of sqlite3 that really encrypts the database. + exists(../../../lib/sqlcipher/.libs/libsqlcipher.a) { + LIBS += ../../../lib/sqlcipher/.libs/libsqlcipher.a + DEPENDPATH += ../../../lib/ + INCLUDEPATH += ../../../lib/ + } else { + error("libsqlcipher is not installed and libsqlcipher.a not found. SQLCIPHER is necessary for encrypted database, to build with unencrypted database, run: qmake CONFIG+=NO_SQLCIPHER") + } } else { - DEPENDPATH += ../../../lib/ - INCLUDEPATH += ../../../lib/ + LIBS *= -lsqlcipher } } diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 00fd3a5e4..9ee2a2ca8 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -85,27 +85,6 @@ linux-* { LIBS += ../../supportlibs/pegmarkdown/lib/libpegmarkdown.a - SQLCIPHER_OK = $$system(pkg-config --exists sqlcipher && echo yes) - isEmpty(SQLCIPHER_OK) { -# We need a explicit path here, to force using the home version of sqlite3 that really encrypts the database. - - exists(../../../lib/sqlcipher/.libs/libsqlcipher.a) { - - LIBS += ../../../lib/sqlcipher/.libs/libsqlcipher.a - DEPENDPATH += ../../../lib/sqlcipher/src/ - INCLUDEPATH += ../../../lib/sqlcipher/src/ - DEPENDPATH += ../../../lib/sqlcipher/tsrc/ - INCLUDEPATH += ../../../lib/sqlcipher/tsrc/ - } else { - message(libsqlcipher.a not found. Compilation will not use SQLCIPHER. Database will be unencrypted.) - DEFINES *= NO_SQLCIPHER - LIBS *= -lsqlite3 - } - - } else { - LIBS += -lsqlcipher - } - LIBS *= -lglib-2.0 LIBS *= -rdynamic DEFINES *= HAVE_XSS # for idle time, libx screensaver extensions diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index fe5c7d6a6..dc18270ba 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -35,27 +35,6 @@ linux-* { LIBS += -lssl -lupnp -lixml -lgnome-keyring LIBS *= -lcrypto -ldl -lz -lpthread LIBS *= -rdynamic - - gxs { - SQLCIPHER_OK = $$system(pkg-config --exists sqlcipher && echo yes) - isEmpty(SQLCIPHER_OK) { -# We need a explicit path here, to force using the home version of sqlite3 that really encrypts the database. - - exists(../../../lib/sqlcipher/.libs/libsqlcipher.a) { - - LIBS += ../../../lib/sqlcipher/.libs/libsqlcipher.a - DEPENDPATH += ../../../lib/sqlcipher/src/ - INCLUDEPATH += ../../../lib/sqlcipher/src/ - } else { - message(libsqlcipher.a not found. Compilation will not use SQLCIPHER. Database will be unencrypted.) - DEFINES *= NO_SQLCIPHER - LIBS *= -lsqlite3 - } - - } else { - LIBS *= -lsqlcipher - } - } } unix { From 829c2f396eb19cde5acb1f92a7ebeb310f63d949 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 20:57:11 +0200 Subject: [PATCH 28/43] Move LIBS from gui/nogui linux section to libretroshare remove -lglib-2.0 it's not needed: was added in d8e327afcc8677579568ad03e4ce040019c5750d to compile for arch, but it compiles on arch now also without this --- libretroshare/src/libretroshare.pro | 6 ++++++ retroshare-gui/src/retroshare-gui.pro | 14 +++----------- retroshare-nogui/src/retroshare-nogui.pro | 10 +--------- 3 files changed, 10 insertions(+), 20 deletions(-) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 6b731a608..c3fea45cd 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -83,6 +83,8 @@ SOURCES += tcponudp/udppeer.cc \ # The next line is for compliance with debian packages. Keep it! INCLUDEPATH += ../libbitdht DEFINES *= RS_USE_BITDHT + PRE_TARGETDEPS *= ../../libbitdht/src/lib/libbitdht.a + LIBS += ../../libbitdht/src/lib/libbitdht.a } @@ -175,6 +177,10 @@ linux-* { DEFINES *= UBUNTU INCLUDEPATH += /usr/include/glib-2.0/ /usr/lib/glib-2.0/include LIBS *= -lgnome-keyring + LIBS *= ../../openpgpsdk/src/lib/libops.a -lbz2 + PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a + LIBS *= -lssl -lupnp -lixml + LIBS *= -lcrypto -lz -lpthread } unix { diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 9ee2a2ca8..fcb7d333c 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -76,17 +76,14 @@ linux-* { QMAKE_CXXFLAGS *= -D_FILE_OFFSET_BITS=64 PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a - PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a LIBS += ../../libretroshare/src/lib/libretroshare.a - LIBS += ../../openpgpsdk/src/lib/libops.a -lbz2 - LIBS += -lssl -lupnp -lixml -lXss -lgnome-keyring - LIBS *= -lcrypto -ldl -lX11 -lz + LIBS *= -lX11 -lXss LIBS += ../../supportlibs/pegmarkdown/lib/libpegmarkdown.a - LIBS *= -lglib-2.0 - LIBS *= -rdynamic + #LIBS *= -lglib-2.0 + LIBS *= -rdynamic -ldl DEFINES *= HAVE_XSS # for idle time, libx screensaver extensions DEFINES *= UBUNTU } @@ -306,11 +303,6 @@ openbsd-* { # ########################################### -bitdht { - LIBS += ../../libbitdht/src/lib/libbitdht.a - PRE_TARGETDEPS *= ../../libbitdht/src/lib/libbitdht.a -} - DEPENDPATH += . ../../libretroshare/src/ INCLUDEPATH += ../../libretroshare/src/ diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index dc18270ba..c8f2f0ca1 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -31,10 +31,7 @@ linux-* { QMAKE_CXXFLAGS *= -D_FILE_OFFSET_BITS=64 LIBS += ../../libretroshare/src/lib/libretroshare.a - LIBS += ../../openpgpsdk/src/lib/libops.a -lbz2 - LIBS += -lssl -lupnp -lixml -lgnome-keyring - LIBS *= -lcrypto -ldl -lz -lpthread - LIBS *= -rdynamic + LIBS *= -rdynamic -ldl } unix { @@ -166,11 +163,6 @@ openbsd-* { ############################## Common stuff ###################################### -# bitdht config -bitdht { - LIBS += ../../libbitdht/src/lib/libbitdht.a -} - DEPENDPATH += . ../../libretroshare/src INCLUDEPATH += . ../../libretroshare/src From 2be400e33ede80c1eb90a1f2829da425c6416af5 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 21:40:52 +0200 Subject: [PATCH 29/43] =?UTF-8?q?CONFIG=20shouldn't=20be=20reset=20at=20qm?= =?UTF-8?q?ake=20call,=20use=20CONFIG+=3D=E2=80=A6=20instead?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With CONFIG=…, the default values of CONFIG are cleared, with qt4 this doesn't seem to be a problem, but with qt5 the build fails. CONFIG=staticlib in openpgpsdk.pro was added in 3caeb0ab7f6912a983dd52934883995d3ea80a92 for OS X, but is not necessary on linux --- README.md | 4 ++-- build_scripts/Debian+Ubuntu/debian/rules | 2 +- build_scripts/RedHat+Fedora/retroshare06.spec | 2 +- openpgpsdk/src/openpgpsdk.pro | 6 +++--- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 617794d77..88d1c5a87 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ Compilation on Linux 3. Compile ```bash cd trunk - qmake CONFIG=debug + qmake CONFIG+=debug make ``` @@ -55,7 +55,7 @@ For packagers ------------- Packagers can use PREFIX and LIB\_DIR to customize the installation paths: ```bash -qmake PREFIX=/usr LIB_DIR=/usr/lib64 +qmake PREFIX=/usr LIB_DIR=/usr/lib64 "CONFIG-=debug" "CONFIG+=release" make make INSTALL_ROOT=${PKGDIR} install ``` diff --git a/build_scripts/Debian+Ubuntu/debian/rules b/build_scripts/Debian+Ubuntu/debian/rules index a0d66eb32..69487e51a 100755 --- a/build_scripts/Debian+Ubuntu/debian/rules +++ b/build_scripts/Debian+Ubuntu/debian/rules @@ -3,7 +3,7 @@ configure: configure-stamp configure-stamp: dh_testdir - cd src && qmake-qt4 CONFIG=release PREFIX=/usr LIB_DIR=/usr/lib RetroShare.pro + cd src && qmake-qt4 "CONFIG-=debug" "CONFIG+=release" PREFIX=/usr LIB_DIR=/usr/lib RetroShare.pro touch $@ diff --git a/build_scripts/RedHat+Fedora/retroshare06.spec b/build_scripts/RedHat+Fedora/retroshare06.spec index bb8339330..d6ef0ff9f 100644 --- a/build_scripts/RedHat+Fedora/retroshare06.spec +++ b/build_scripts/RedHat+Fedora/retroshare06.spec @@ -56,7 +56,7 @@ cd lib/sqlcipher make cd - cd src -qmake-qt4 CONFIG=release PREFIX=%{_prefix} LIB_DIR=%{_libdir} RetroShare.pro +qmake-qt4 "CONFIG-=debug" "CONFIG+=release" PREFIX=%{_prefix} LIB_DIR=%{_libdir} RetroShare.pro make cd - diff --git a/openpgpsdk/src/openpgpsdk.pro b/openpgpsdk/src/openpgpsdk.pro index 786ee5819..f54f5d7f8 100644 --- a/openpgpsdk/src/openpgpsdk.pro +++ b/openpgpsdk/src/openpgpsdk.pro @@ -1,10 +1,10 @@ !include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") TEMPLATE = lib -win32 { - CONFIG += staticlib -} else { +macx { CONFIG = staticlib +} else { + CONFIG += staticlib } DEFINES *= OPENSSL_NO_IDEA From 30d7aa3be33b8f575773e5c88a14bca5d1eeee34 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 22:08:09 +0200 Subject: [PATCH 30/43] Remove dependency on qt script, it's unused --- retroshare-gui/src/retroshare-gui.pro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index fcb7d333c..8e3cff862 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -1,6 +1,6 @@ !include("../../retroshare.pri"): error("Could not include file ../../retroshare.pri") -QT += network xml script +QT += network xml CONFIG += qt gui uic qrc resources idle bitdht CONFIG += link_prl From d116f40d6ea9a7c992b0deda0f472f6ac6b27eb2 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 23:06:31 +0200 Subject: [PATCH 31/43] Move openpgpsdk to common section of libretroshare.pro It is the same for all platforms --- libretroshare/src/libretroshare.pro | 6 ++++-- retroshare-gui/src/retroshare-gui.pro | 5 ----- retroshare-nogui/src/retroshare-nogui.pro | 5 ----- 3 files changed, 4 insertions(+), 12 deletions(-) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index c3fea45cd..92c8127d2 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -177,8 +177,6 @@ linux-* { DEFINES *= UBUNTU INCLUDEPATH += /usr/include/glib-2.0/ /usr/lib/glib-2.0/include LIBS *= -lgnome-keyring - LIBS *= ../../openpgpsdk/src/lib/libops.a -lbz2 - PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a LIBS *= -lssl -lupnp -lixml LIBS *= -lcrypto -lz -lpthread } @@ -341,6 +339,10 @@ openbsd-* { ################################### COMMON stuff ################################## +# openpgpsdk +PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a +LIBS *= ../../openpgpsdk/src/lib/libops.a -lbz2 + HEADERS += dbase/cachestrapper.h \ dbase/fimonitor.h \ dbase/findex.h \ diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 8e3cff862..0108fb7f6 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -190,12 +190,10 @@ win32 { #QTPLUGIN += qjpeg PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a - PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a LIBS_DIR = $$PWD/../../../libs LIBS += ../../libretroshare/src/lib/libretroshare.a - LIBS += ../../openpgpsdk/src/lib/libops.a -lbz2 LIBS += -L"$$LIBS_DIR/lib" LIBS += ../../supportlibs/pegmarkdown/lib/libpegmarkdown.a @@ -240,7 +238,6 @@ macx { CONFIG += version_detail_bash_script LIBS += ../../libretroshare/src/lib/libretroshare.a - LIBS += ../../openpgpsdk/src/lib/libops.a -lbz2 LIBS += -lssl -lcrypto -lz #LIBS += -lssl -lcrypto -lz -lgpgme -lgpg-error -lassuan LIBS += ../../../miniupnpc-1.0/libminiupnpc.a @@ -278,10 +275,8 @@ openbsd-* { INCLUDEPATH *= /usr/local/include PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a - PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a LIBS *= ../../libretroshare/src/lib/libretroshare.a - LIBS *= ../../openpgpsdk/src/lib/libops.a -lbz2 LIBS *= -lssl -lcrypto LIBS *= -lgpgme LIBS *= -lupnp diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index c8f2f0ca1..d5438515c 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -76,12 +76,10 @@ win32 { MOC_DIR = temp/moc PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a - PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a LIBS_DIR = $$PWD/../../../libs LIBS += ../../libretroshare/src/lib/libretroshare.a - LIBS += ../../openpgpsdk/src/lib/libops.a -lbz2 LIBS += -L"$$LIBS_DIR/lib" LIBS += -lssl -lcrypto -lpthread -lminiupnpc -lz # added after bitdht @@ -111,7 +109,6 @@ macx { LIBS += -Wl,-search_paths_first LIBS += ../../libretroshare/src/lib/libretroshare.a - LIBS += ../../openpgpsdk/src/lib/libops.a -lbz2 LIBS += -lssl -lcrypto -lz LIBS += ../../../miniupnpc-1.0/libminiupnpc.a LIBS += -framework CoreFoundation @@ -150,13 +147,11 @@ openbsd-* { INCLUDEPATH *= /usr/local/include QMAKE_CXXFLAGS *= -Dfseeko64=fseeko -Dftello64=ftello -Dstat64=stat -Dstatvfs64=statvfs -Dfopen64=fopen LIBS *= ../../libretroshare/src/lib/libretroshare.a - LIBS *= ../../openpgpsdk/src/lib/libops.a -lbz2 LIBS *= -lssl -lcrypto LIBS *= -lgpgme LIBS *= -lupnp LIBS *= -lgnome-keyring PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a - PRE_TARGETDEPS *= ../../openpgpsdk/src/lib/libops.a LIBS *= -rdynamic } From 069b112135bb8418f5169bbc0ad359c66049f825 Mon Sep 17 00:00:00 2001 From: AsamK Date: Mon, 31 Aug 2015 23:15:17 +0200 Subject: [PATCH 32/43] Update README.md --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 88d1c5a87..4072f6a3f 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,17 @@ Compilation on Linux /usr/bin/RetroShare06 /usr/bin/RetroShare06-nogui +Compile only retroshare-nogui +----------------------------- +If you want to run RetroShare on a server and don’t need the gui and plugins, +you can run the following commands to only compile/install the nogui version: + +```bash +qmake +make retroshare-nogui +sudo make retroshare-nogui-install_subtargets +``` + For packagers ------------- Packagers can use PREFIX and LIB\_DIR to customize the installation paths: From dda8a49ef51226aee4738287045dded75bb731c3 Mon Sep 17 00:00:00 2001 From: AsamK Date: Tue, 1 Sep 2015 14:50:55 +0200 Subject: [PATCH 33/43] Add CONFIG+=NO_SQLCIPHER to travis config --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6759fd515..aecd47c39 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,12 +27,12 @@ addons: project: name: "RetroShare/RetroShare" description: "RetroShare Build submitted via Travis CI" - build_command_prepend: "qmake; make clean" + build_command_prepend: "qmake CONFIG+=NO_SQLCIPHER; make clean" build_command: "make -j 4" branch_pattern: coverity_scan before_script: - - qmake + - qmake CONFIG+=NO_SQLCIPHER #script: make script: if [ "${COVERITY_SCAN_BRANCH}" != 1 ]; then make ; fi From c3fff396249abd85d7f0fd0efe3e294dca473675 Mon Sep 17 00:00:00 2001 From: thunder2 Date: Tue, 1 Sep 2015 22:49:08 +0200 Subject: [PATCH 34/43] Fixed Windows compile --- libresapi/src/api/ForumHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libresapi/src/api/ForumHandler.cpp b/libresapi/src/api/ForumHandler.cpp index 6d312958f..3a628a5a5 100644 --- a/libresapi/src/api/ForumHandler.cpp +++ b/libresapi/src/api/ForumHandler.cpp @@ -105,7 +105,7 @@ void ForumHandler::handleWildcard(Request &req, Response &resp) { RsGxsForumGroup& grp = *vit; KeyValueReference id("id", grp.mMeta.mGroupId); - KeyValueReference vis_msg("visible_msg_count", grp.mMeta.mVisibleMsgCount); + KeyValueReference vis_msg("visible_msg_count", grp.mMeta.mVisibleMsgCount); //KeyValueReference pgp_id("pgp_id",grp.mPgpId ); // not very happy about this, i think the flags should stay hidden in rsidentities bool own = (grp.mMeta.mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_ADMIN); From 3e6874199410925b05cb18ba88ee0d479f83a2a7 Mon Sep 17 00:00:00 2001 From: Phenom Date: Tue, 1 Sep 2015 22:49:15 +0200 Subject: [PATCH 35/43] Fix Windows compilation of libresapi with error: \libresapi\src\api\ForumHandler.cpp:108: erreur : 'u_int32_t' was not declared in this scope KeyValueReference vis_msg("visible_msg_count", grp.mMeta.mVisibleMsgCount); ^ --- libresapi/src/api/ForumHandler.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libresapi/src/api/ForumHandler.cpp b/libresapi/src/api/ForumHandler.cpp index 6d312958f..1a3a9c5c0 100644 --- a/libresapi/src/api/ForumHandler.cpp +++ b/libresapi/src/api/ForumHandler.cpp @@ -8,6 +8,9 @@ #include "GxsResponseTask.h" #ifndef WINDOWS_SYS #include "unistd.h" +#else +#include "stdint.h" +typedef uint32_t u_int32_t; #endif namespace resource_api From 3cfe69787f0c7b59ce0ee42aa246c9c02233ec42 Mon Sep 17 00:00:00 2001 From: AsamK Date: Wed, 2 Sep 2015 02:03:46 +0200 Subject: [PATCH 36/43] Fix subtle bug in tlv deserialization of TLV_TYPE_KEYSIGNATURETYPE When using -O2, the variable currType is optimized out, so the signature is added with the wrong sign type. Then when verifying the data the signature is not found and verification fails. The main sympton of this bug was receiving no more forum/channel posts. Wit debug builds (-O0) this worked, that's why not every one was affected by this. This bug was introduced in 2012 (19e856c2a8eb56c5e43e977aac27a3dbc79ab2d0) --- libretroshare/src/serialiser/rstlvkeys.cc | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/libretroshare/src/serialiser/rstlvkeys.cc b/libretroshare/src/serialiser/rstlvkeys.cc index eb90fff52..968b09d83 100644 --- a/libretroshare/src/serialiser/rstlvkeys.cc +++ b/libretroshare/src/serialiser/rstlvkeys.cc @@ -649,7 +649,6 @@ bool RsTlvKeySignatureSet::GetTlv(void *data, uint32_t size, uint32_t *offset) /* get the next type */ uint16_t tlvsubtype = GetTlvType( &(((uint8_t *) data)[*offset]) ); - SignType currType; switch(tlvsubtype) { @@ -659,16 +658,13 @@ bool RsTlvKeySignatureSet::GetTlv(void *data, uint32_t size, uint32_t *offset) ok &= sign.GetTlv(data, size, offset); if (ok) { - keySignSet[currType] = sign; + keySignSet[sign_type] = sign; } } break; case TLV_TYPE_KEYSIGNATURETYPE: { ok = GetTlvUInt32(data, size, offset, TLV_TYPE_KEYSIGNATURETYPE, &sign_type); - - if(ok) - currType = sign_type; } break; default: From 6af57f3aeac5171782cba5d11b2ac9e39931a6c2 Mon Sep 17 00:00:00 2001 From: Cyril Soler Date: Tue, 1 Sep 2015 21:03:23 -0400 Subject: [PATCH 37/43] Revert "Fix Windows compilation of libresapi with error:" --- libresapi/src/api/ForumHandler.cpp | 3 --- 1 file changed, 3 deletions(-) diff --git a/libresapi/src/api/ForumHandler.cpp b/libresapi/src/api/ForumHandler.cpp index 144a61057..3a628a5a5 100644 --- a/libresapi/src/api/ForumHandler.cpp +++ b/libresapi/src/api/ForumHandler.cpp @@ -8,9 +8,6 @@ #include "GxsResponseTask.h" #ifndef WINDOWS_SYS #include "unistd.h" -#else -#include "stdint.h" -typedef uint32_t u_int32_t; #endif namespace resource_api From 11d54b8c0ba2222ac26e2fa8fe012afc034fbcf9 Mon Sep 17 00:00:00 2001 From: Chozabu Date: Wed, 2 Sep 2015 22:37:34 +0100 Subject: [PATCH 38/43] added more details to travis IRC output --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index aecd47c39..d2d2c2781 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,6 +53,8 @@ notifications: - "chat.freenode.net#retroshare" template: - "%{repository}/%{branch} (%{commit} - %{author}): %{build_url}: %{message}" + - "Message: %{commit_message}" + - "Commit details: %{compare_url}" # webhooks: # urls: # - https://webhooks.gitter.im/e/9502afd22ca6c8e85fb3 From feafb1e2914f5d2d3ed6422fd7ab0b6f0470238a Mon Sep 17 00:00:00 2001 From: AsamK Date: Thu, 3 Sep 2015 00:27:14 +0200 Subject: [PATCH 39/43] Add "Paste as plain text" to context menu of RS text edits Useful for example when pasting from a website --- retroshare-gui/src/gui/common/MimeTextEdit.cpp | 8 ++++++++ retroshare-gui/src/gui/common/MimeTextEdit.h | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/common/MimeTextEdit.cpp b/retroshare-gui/src/gui/common/MimeTextEdit.cpp index 1c51980c4..9513703d6 100644 --- a/retroshare-gui/src/gui/common/MimeTextEdit.cpp +++ b/retroshare-gui/src/gui/common/MimeTextEdit.cpp @@ -19,6 +19,8 @@ * Boston, MA 02110-1301, USA. ****************************************************************/ +#include +#include #include #include #include @@ -228,6 +230,7 @@ void MimeTextEdit::contextMenuEvent(QContextMenuEvent *e) QMenu *contextMenu = createStandardContextMenu(e->pos()); /* Add actions for pasting links */ + contextMenu->addAction( tr("Paste as plain text"), this, SLOT(pastePlainText())); contextMenu->addSeparator(); QAction *pasteLinkAction = contextMenu->addAction(QIcon(":/images/pasterslink.png"), tr("Paste RetroShare Link"), this, SLOT(pasteLink())); contextMenu->addAction(QIcon(":/images/pasterslink.png"), tr("Paste my certificate link"), this, SLOT(pasteOwnCertificateLink())); @@ -260,3 +263,8 @@ void MimeTextEdit::pasteOwnCertificateLink() insertHtml(link.toHtml() + " "); } } + +void MimeTextEdit::pastePlainText() +{ + insertPlainText(QApplication::clipboard()->text()); +} diff --git a/retroshare-gui/src/gui/common/MimeTextEdit.h b/retroshare-gui/src/gui/common/MimeTextEdit.h index 5ef5d6a5f..d28bb2c6e 100644 --- a/retroshare-gui/src/gui/common/MimeTextEdit.h +++ b/retroshare-gui/src/gui/common/MimeTextEdit.h @@ -58,7 +58,7 @@ private slots: void insertCompletion(const QString &completion); void pasteLink(); void pasteOwnCertificateLink(); - + void pastePlainText(); private: QString textUnderCursor() const; From 77c0d562fb5c450937de0f9054c39add4f67f5ae Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 2 Sep 2015 22:14:04 -0400 Subject: [PATCH 40/43] fixed deleting/stopping video devices --- plugins/VOIP/gui/AudioInputConfig.cpp | 10 +++++++--- plugins/VOIP/gui/QVideoDevice.cpp | 14 +++++++++++++- plugins/VOIP/gui/QVideoDevice.h | 4 ++-- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index 6ef39bce5..ea0f2e129 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -157,6 +157,8 @@ void AudioInputConfig::togglePreview(bool b) AudioInputConfig::~AudioInputConfig() { + disconnect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) ); + graph_source->stop() ; graph_source->setVideoInput(NULL) ; @@ -350,8 +352,10 @@ void AudioInputConfig::on_qcbTransmit_currentIndexChanged(int v) { } -void AudioInputConfig::on_Tick_timeout() { - if (!inputAudioProcessor) { +void AudioInputConfig::on_Tick_timeout() +{ + if (!inputAudioProcessor) + { inputAudioProcessor = new QtSpeex::SpeexInputProcessor(); inputAudioProcessor->open(QIODevice::WriteOnly | QIODevice::Unbuffered); @@ -376,7 +380,7 @@ void AudioInputConfig::on_Tick_timeout() { // also transmit encoded video RsVOIPDataChunk chunk ; - while(videoInput->getNextEncodedPacket(chunk)) + while((!videoInput->stopped()) && videoInput->getNextEncodedPacket(chunk)) { videoProcessor->receiveEncodedData(chunk) ; chunk.clear() ; diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index 3a0cc4aa9..b81b7b79c 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -15,6 +15,11 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent) _echo_output_device = NULL ; } +bool QVideoInputDevice::stopped() +{ + return _timer == NULL ; +} + void QVideoInputDevice::stop() { if(_timer != NULL) @@ -54,6 +59,9 @@ void QVideoInputDevice::start() void QVideoInputDevice::grabFrame() { + if(!_timer) + return ; + IplImage *img=cvQueryFrame(_capture_device); if(img == NULL) @@ -87,6 +95,9 @@ void QVideoInputDevice::grabFrame() bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { + if(!_timer) + return false ; + if(_video_processor) return _video_processor->nextEncodedPacket(chunk) ; else @@ -100,7 +111,8 @@ uint32_t QVideoInputDevice::currentBandwidth() const QVideoInputDevice::~QVideoInputDevice() { - stop() ; + stop() ; + _video_processor = NULL ; } diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index 20a837446..cc92302a2 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -51,8 +51,8 @@ class QVideoInputDevice: public QObject void start() ; void stop() ; - - protected slots: + bool stopped(); +protected slots: void grabFrame() ; signals: From b1b2de617a7e1c4693eafe495c6b2c42562c1130 Mon Sep 17 00:00:00 2001 From: thunder2 Date: Thu, 3 Sep 2015 13:11:12 +0200 Subject: [PATCH 41/43] Fixed Windows compile. - Set name for prl file in libretroshare.pro - Use "QMAKE_LFLAGS += -Wl,--start-group" to solve link issue --- libretroshare/src/libretroshare.pro | 1 + retroshare-gui/src/retroshare-gui.pro | 12 ++++-------- retroshare-nogui/src/retroshare-nogui.pro | 10 ++++------ 3 files changed, 9 insertions(+), 14 deletions(-) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 92c8127d2..18ded24a5 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -5,6 +5,7 @@ CONFIG += staticlib bitdht CONFIG += create_prl CONFIG -= qt TARGET = retroshare +TARGET_PRL = libretroshare #GXS Stuff. diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 0108fb7f6..9727227e1 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -175,6 +175,9 @@ win32 { QMAKE_CFLAGS += -Wextra QMAKE_CXXFLAGS += -Wextra + # solve linker warnings because of the order of the libraries + QMAKE_LFLAGS += -Wl,--start-group + # Switch off optimization for release version QMAKE_CXXFLAGS_RELEASE -= -O2 QMAKE_CXXFLAGS_RELEASE += -O0 @@ -199,9 +202,7 @@ win32 { LIBS += ../../supportlibs/pegmarkdown/lib/libpegmarkdown.a LIBS += -lsqlcipher - LIBS += -lssl -lcrypto -lpthread -lminiupnpc -lz -# added after bitdht -# LIBS += -lws2_32 + LIBS += -lssl -lcrypto -lpthread -lminiupnpc -lz -lws2_32 LIBS += -luuid -lole32 -liphlpapi -lcrypt32 -lgdi32 LIBS += -lole32 -lwinmm RC_FILE = gui/images/retroshare_win.rc @@ -307,11 +308,6 @@ INCLUDEPATH += ../../libresapi/src PRE_TARGETDEPS *= ../../libresapi/src/lib/libresapi.a LIBS += ../../libresapi/src/lib/libresapi.a -lmicrohttpd -win32 { -# must be added after bitdht - LIBS += -lws2_32 -} - # Input HEADERS += rshare.h \ retroshare-gui/configpage.h \ diff --git a/retroshare-nogui/src/retroshare-nogui.pro b/retroshare-nogui/src/retroshare-nogui.pro index d5438515c..a15cb1bb2 100644 --- a/retroshare-nogui/src/retroshare-nogui.pro +++ b/retroshare-nogui/src/retroshare-nogui.pro @@ -75,6 +75,9 @@ win32 { UI_DIR = temp/ui MOC_DIR = temp/moc + # solve linker warnings because of the order of the libraries + QMAKE_LFLAGS += -Wl,--start-group + PRE_TARGETDEPS *= ../../libretroshare/src/lib/libretroshare.a LIBS_DIR = $$PWD/../../../libs @@ -82,8 +85,7 @@ win32 { LIBS += ../../libretroshare/src/lib/libretroshare.a LIBS += -L"$$LIBS_DIR/lib" LIBS += -lssl -lcrypto -lpthread -lminiupnpc -lz -# added after bitdht -# LIBS += -lcrypto -lws2_32 -lgdi32 + LIBS += -lcrypto -lws2_32 -lgdi32 LIBS += -luuid -lole32 -liphlpapi -lcrypt32 LIBS += -lole32 -lwinmm @@ -353,7 +355,3 @@ protorpc { INCLUDEPATH += $${PROTOPATH}/src } } -win32 { -# must be added after ssh - LIBS += -lcrypto -lws2_32 -lgdi32 -} From 9bbdefa48aa56b4777eadeee05ad177c9eedd433 Mon Sep 17 00:00:00 2001 From: thunder2 Date: Thu, 3 Sep 2015 16:24:57 +0200 Subject: [PATCH 42/43] Added "QMAKE_LFLAGS += -Wl,--start-group" to pluigns for Windows compile. --- plugins/Common/retroshare_plugin.pri | 3 +++ plugins/FeedReader/FeedReader.pro | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/plugins/Common/retroshare_plugin.pri b/plugins/Common/retroshare_plugin.pri index d20c1baa4..3c9644c55 100644 --- a/plugins/Common/retroshare_plugin.pri +++ b/plugins/Common/retroshare_plugin.pri @@ -28,6 +28,9 @@ win32 { QMAKE_CFLAGS += -Wextra QMAKE_CXXFLAGS += -Wextra + # solve linker warnings because of the order of the libraries + QMAKE_LFLAGS += -Wl,--start-group + OBJECTS_DIR = temp/obj MOC_DIR = temp/moc RCC_DIR = temp/qrc diff --git a/plugins/FeedReader/FeedReader.pro b/plugins/FeedReader/FeedReader.pro index acc3b4e4c..4cebc5f48 100644 --- a/plugins/FeedReader/FeedReader.pro +++ b/plugins/FeedReader/FeedReader.pro @@ -92,8 +92,7 @@ linux-* { win32 { DEFINES += CURL_STATICLIB LIBXML_STATIC LIBXSLT_STATIC LIBEXSLT_STATIC - # Change order of the libraries - LIBS = -lcurl -lxml2 -lz -lxslt -lws2_32 -lwldap32 -lssl -lcrypto -lgdi32 -lwsock32 $${LIBS} + LIBS += -lcurl -lxml2 -lz -lxslt -lws2_32 -lwldap32 -lssl -lcrypto } openbsd-* { From 738dadadc9055698b4159c407d5ad779104326ff Mon Sep 17 00:00:00 2001 From: thunder2 Date: Thu, 3 Sep 2015 16:33:50 +0200 Subject: [PATCH 43/43] Fixed Windows compile of VOIP. --- plugins/VOIP/VOIP.pro | 3 +++ plugins/VOIP/gui/VideoProcessor.cpp | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/plugins/VOIP/VOIP.pro b/plugins/VOIP/VOIP.pro index 86a716309..e560a8501 100644 --- a/plugins/VOIP/VOIP.pro +++ b/plugins/VOIP/VOIP.pro @@ -24,6 +24,9 @@ linux-* { } win32 { + # ffmpeg + QMAKE_CXXFLAGS += -D__STDC_CONSTANT_MACROS + LIBS_DIR = $$PWD/../../../libs LIBS += -L"$$LIBS_DIR/lib/opencv" diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index a0084ea18..20aca0410 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -10,7 +10,11 @@ #include "QVideoDevice.h" #include +#include +#if defined(__MINGW32__) +#define memalign _aligned_malloc +#endif //MINGW extern "C" { #include