Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-10-01 02:35:48 -04:00
Code Issues Actions Packages Projects Releases Wiki Activity

merged Pull Request #81, providing avcodec video encoding/decoding

Browse Source
This commit is contained in:
csoler 2015-09-02 22:39:39 -04:00
commit 5dfdffb763
17 changed files with 1109 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build_scripts/Debian+Ubuntu/control.precise
View File

@ -2,7 +2,7 @@ Source: retroshare06
Section: devel
Priority: standard
Maintainer: Cyril Soler <csoler@users.sourceforge.net>
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libopencv-core-dev, libopencv-contrib-dev, libhighgui-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev
Standards-Version: 3.9.3
Homepage: http://retroshare.sourceforge.net

2
build_scripts/Debian+Ubuntu/control.squeeze_bubba3
View File

@ -2,7 +2,7 @@ Source: retroshare06
Section: devel
Priority: standard
Maintainer: Cyril Soler <csoler@users.sourceforge.net>
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqt4-multimedia, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libcv-dev, libcvaux-dev, libhighgui-dev, tcl8.5, libmicrohttpd-dev, libsqlite3-dev, libavcodec-dev
Standards-Version: 3.9.3
Homepage: http://retroshare.sourceforge.net

2
build_scripts/Debian+Ubuntu/control.ubuntu_lucid
View File

@ -2,7 +2,7 @@ Source: retroshare
Section: devel
Priority: standard
Maintainer: Cyril Soler <csoler@users.sourceforge.net>
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, libprotobuf-dev, protobuf-compiler, cmake, libcurl4-openssl-dev, libavcodec-dev
Standards-Version: 3.9.1
Homepage: http://retroshare.sourceforge.net

2
build_scripts/Debian+Ubuntu/debian/control
View File

@ -2,7 +2,7 @@ Source: retroshare06
Section: devel
Priority: standard
Maintainer: Cyril Soler <csoler@users.sourceforge.net>
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev
Build-Depends: debhelper (>= 7), libglib2.0-dev, libupnp-dev, qt4-dev-tools, libqt4-dev, libssl-dev, libxss-dev, libgnome-keyring-dev, libbz2-dev, libqt4-opengl-dev, libqtmultimediakit1, qtmobility-dev, libspeex-dev, libspeexdsp-dev, libxslt1-dev, cmake, libcurl4-openssl-dev, libopencv-dev, tcl8.5, libsqlcipher-dev, libmicrohttpd-dev, libavcodec-dev
Standards-Version: 3.9.3
Homepage: http://retroshare.sourceforge.net

2
plugins/VOIP/VOIP.pro
View File

@ -96,4 +96,4 @@ TRANSLATIONS += \
lang/VOIP_tr.ts \
lang/VOIP_zh_CN.ts
LIBS += -lspeex -lspeexdsp
LIBS += -lspeex -lspeexdsp -lavformat -lavcodec -lavutil

101
plugins/VOIP/gui/AudioInputConfig.cpp
View File

@ -29,17 +29,15 @@
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <QTimer>
//#include "AudioInput.h"
//#include "AudioOutput.h"
#include "AudioStats.h"
#include "AudioInputConfig.h"
//#include "Global.h"
//#include "NetworkConfig.h"
#include "audiodevicehelper.h"
#include "AudioWizard.h"
#include "gui/VideoProcessor.h"
#include "gui/common/RSGraphWidget.h"
#include "util/RsProtectedTimer.h"
#include <interface/rsVOIP.h>
#define iroundf(x) ( static_cast<int>(x) )
@ -55,9 +53,9 @@ void AudioInputDialog::showEvent(QShowEvent *) {
class voipGraphSource: public RSGraphSource
{
public:
voipGraphSource() {}
voipGraphSource() : video_input(NULL) {}
void setVideoInput(QVideoInputDevice *vid) { video_input = vid ; }
void setVideoInput(const QVideoInputDevice *vid) { video_input = vid ; }
virtual QString displayName(int) const { return tr("Required bandwidth") ;}
@ -72,22 +70,15 @@ public:
}
virtual void getValues(std::map<std::string,float>& vals) const
{
RsVOIPDataChunk chunk ;
uint32_t total_size = 0 ;
{
vals.clear() ;
while(video_input && video_input->getNextEncodedPacket(chunk))
{
total_size += chunk.size ;
chunk.clear() ;
}
vals[std::string("bw")] = (float)total_size ;
if(video_input)
vals[std::string("bw")] = video_input->currentBandwidth() ;
}
private:
QVideoInputDevice *video_input ;
const QVideoInputDevice *video_input ;
};
void voipGraph::setVoipSource(voipGraphSource *gs)
@ -109,7 +100,7 @@ voipGraph::voipGraph(QWidget *parent)
AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags)
: ConfigPage(parent, flags)
{
std::cerr << "Creating audioInputConfig object" << std::endl;
std::cerr << "Creating audioInputConfig object" << std::endl;
/* Invoke the Qt Designer generated object setup routine */
ui.setupUi(this);
@ -121,22 +112,53 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags)
abSpeech = NULL;
qtTick = NULL;
// Create the video pipeline.
//
videoInput = new QVideoInputDevice(this) ;
videoInput->setEchoVideoTarget(ui.videoDisplay) ;
videoInput->setVideoEncoder(new JPEGVideoEncoder()) ;
graph_source = new voipGraphSource ;
ui.voipBwGraph->setSource(graph_source);
graph_source->setVideoInput(videoInput) ;
graph_source->setCollectionTimeLimit(1000*300) ;
graph_source->start() ;
// Create the video pipeline.
//
videoInput = new QVideoInputDevice(this) ;
videoInput->setEchoVideoTarget(ui.videoDisplay) ;
videoProcessor = new VideoProcessor() ;
videoProcessor->setDisplayTarget(NULL) ;
videoProcessor->setMaximumBandwidth(ui.availableBW_SB->value()) ;
videoInput->setVideoProcessor(videoProcessor) ;
graph_source = new voipGraphSource ;
ui.voipBwGraph->setSource(graph_source);
graph_source->setVideoInput(videoInput) ;
graph_source->setCollectionTimeLimit(1000*300) ;
graph_source->start() ;
QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ;
QObject::connect(ui.availableBW_SB,SIGNAL(valueChanged(double)),this,SLOT(updateAvailableBW(double))) ;
}
void AudioInputConfig::updateAvailableBW(double r)
{
std::cerr << "Setting max bandwidth to " << r << " KB/s" << std::endl;
videoProcessor->setMaximumBandwidth((uint32_t)(r*1024)) ;
}
void AudioInputConfig::togglePreview(bool b)
{
if(b)
{
videoInput->setEchoVideoTarget(NULL) ;
videoProcessor->setDisplayTarget(ui.videoDisplay) ;
}
else
{
videoInput->setEchoVideoTarget(ui.videoDisplay) ;
videoProcessor->setDisplayTarget(NULL) ;
}
}
AudioInputConfig::~AudioInputConfig()
{
disconnect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) );
graph_source->stop() ;
graph_source->setVideoInput(NULL) ;
@ -166,7 +188,7 @@ void AudioInputConfig::load()
//connect( ui.allowIpDeterminationCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleIpDetermination(bool) ) );
//connect( ui.allowTunnelConnectionCB, SIGNAL( toggled( bool ) ), this, SLOT( toggleTunnelConnection(bool) ) );
qtTick = new QTimer(this);
qtTick = new RsProtectedTimer(this);
connect( qtTick, SIGNAL( timeout ( ) ), this, SLOT( on_Tick_timeout() ) );
qtTick->start(20);
/*if (AudioInputRegistrar::qmNew) {
@ -330,8 +352,10 @@ void AudioInputConfig::on_qcbTransmit_currentIndexChanged(int v) {
}
void AudioInputConfig::on_Tick_timeout() {
if (!inputAudioProcessor) {
void AudioInputConfig::on_Tick_timeout()
{
if (!inputAudioProcessor)
{
inputAudioProcessor = new QtSpeex::SpeexInputProcessor();
inputAudioProcessor->open(QIODevice::WriteOnly | QIODevice::Unbuffered);
@ -352,6 +376,15 @@ void AudioInputConfig::on_Tick_timeout() {
abSpeech->iValue = iroundf(inputAudioProcessor->dVoiceAcivityLevel * 32767.0f + 0.5f);
abSpeech->update();
// also transmit encoded video
RsVOIPDataChunk chunk ;
while((!videoInput->stopped()) && videoInput->getNextEncodedPacket(chunk))
{
videoProcessor->receiveEncodedData(chunk) ;
chunk.clear() ;
}
}
void AudioInputConfig::emptyBuffer() {

7
plugins/VOIP/gui/AudioInputConfig.h
View File

@ -69,6 +69,7 @@ class AudioInputConfig : public ConfigPage
//VideoDecoder *videoDecoder ;
//VideoEncoder *videoEncoder ;
QVideoInputDevice *videoInput ;
VideoProcessor *videoProcessor ;
bool loaded;
voipGraphSource *graph_source ;
@ -92,10 +93,12 @@ class AudioInputConfig : public ConfigPage
virtual QPixmap iconPixmap() const { return QPixmap(":/images/talking_on.svg") ; }
virtual QString pageName() const { return tr("VOIP") ; }
virtual QString helpText() const { return ""; }
private slots:
private slots:
void updateAvailableBW(double r);
void loadSettings();
void emptyBuffer();
void togglePreview(bool) ;
void on_qsTransmitHold_valueChanged(int v);
void on_qsAmp_valueChanged(int v);

45
plugins/VOIP/gui/AudioInputConfig.ui
View File

@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>1155</width>
<height>713</height>
<height>832</height>
</rect>
</property>
<layout class="QVBoxLayout" name="verticalLayout">
@ -391,6 +391,49 @@
</layout>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QLabel" name="label_2">
<property name="text">
<string>Available bandwidth:</string>
</property>
</widget>
</item>
<item>
<widget class="QDoubleSpinBox" name="availableBW_SB">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Use this field to simulate the maximum bandwidth available so as to preview what the encoded video will look like with the corresponding compression rate.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="suffix">
<string>KB/s</string>
</property>
<property name="decimals">
<number>1</number>
</property>
<property name="minimum">
<double>2.000000000000000</double>
</property>
<property name="maximum">
<double>200.000000000000000</double>
</property>
<property name="value">
<double>30.000000000000000</double>
</property>
</widget>
</item>
</layout>
</item>
<item>
<widget class="QCheckBox" name="showEncoded_CB">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Display encoded (and then decoded) frame, to check the codec's quality. If not selected, the image above only shows the frame that is grabbed from your camera.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="text">
<string>preview</string>
</property>
</widget>
</item>
<item>
<spacer>
<property name="orientation">

71
plugins/VOIP/gui/QVideoDevice.cpp
View File

@ -11,10 +11,15 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent)
{
_timer = NULL ;
_capture_device = NULL ;
_video_encoder = NULL ;
_video_processor = NULL ;
_echo_output_device = NULL ;
}
bool QVideoInputDevice::stopped()
{
return _timer == NULL ;
}
void QVideoInputDevice::stop()
{
if(_timer != NULL)
@ -54,49 +59,60 @@ void QVideoInputDevice::start()
void QVideoInputDevice::grabFrame()
{
IplImage *img=cvQueryFrame(_capture_device);
if(!_timer)
return ;
IplImage *img=cvQueryFrame(_capture_device);
if(img == NULL)
{
std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl;
return ;
}
// get the image data
if(img == NULL)
{
std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl;
return ;
}
// get the image data
if(img->nChannels != 3)
{
std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl;
return ;
}
if(img->nChannels != 3)
{
std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl;
return ;
}
// convert to RGB and copy to new buffer, because cvQueryFrame tells us to not modify the buffer
cv::Mat img_rgb;
cv::cvtColor(cv::Mat(img), img_rgb, CV_BGR2RGB);
static const int _encoded_width = 128 ;
static const int _encoded_height = 128 ;
QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888);
QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888).scaled(QSize(_encoded_width,_encoded_height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ;
if(_video_processor != NULL)
{
_video_processor->processImage(image) ;
if(_video_encoder != NULL)
{
_video_encoder->addImage(image) ;
emit networkPacketReady() ;
}
if(_echo_output_device != NULL) _echo_output_device->showFrame(image) ;
emit networkPacketReady() ;
}
if(_echo_output_device != NULL)
_echo_output_device->showFrame(image) ;
}
bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk)
{
if(_video_encoder)
return _video_encoder->nextPacket(chunk) ;
if(!_timer)
return false ;
if(_video_processor)
return _video_processor->nextEncodedPacket(chunk) ;
else
return false ;
return false ;
}
uint32_t QVideoInputDevice::currentBandwidth() const
{
return _video_processor->currentBandwidthOut() ;
}
QVideoInputDevice::~QVideoInputDevice()
{
stop() ;
stop() ;
_video_processor = NULL ;
}
@ -113,6 +129,7 @@ void QVideoOutputDevice::showFrameOff()
void QVideoOutputDevice::showFrame(const QImage& img)
{
setPixmap(QPixmap::fromImage(img).scaled(minimumSize(),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ;
std::cerr << "img.size = " << img.width() << " x " << img.height() << std::endl;
setPixmap(QPixmap::fromImage(img).scaled( QSize(height()*640/480,height()),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ;
}

15
plugins/VOIP/gui/QVideoDevice.h
View File

@ -2,6 +2,7 @@
#include <QLabel>
#include "interface/rsVOIP.h"
#include "gui/VideoProcessor.h"
class VideoEncoder ;
class CvCapture ;
@ -31,7 +32,7 @@ class QVideoInputDevice: public QObject
// Captured images are sent to this encoder. Can be NULL.
//
void setVideoEncoder(VideoEncoder *venc) { _video_encoder = venc ; }
void setVideoProcessor(VideoProcessor *venc) { _video_processor = venc ; }
// All images received will be echoed to this target. We could use signal/slots, but it's
// probably faster this way. Can be NULL.
@ -42,17 +43,23 @@ class QVideoInputDevice: public QObject
//
bool getNextEncodedPacket(RsVOIPDataChunk&) ;
// gets the estimated current bandwidth required to transmit the encoded data, in B/s
//
uint32_t currentBandwidth() const ;
// control
void start() ;
void stop() ;
protected slots:
bool stopped();
protected slots:
void grabFrame() ;
signals:
void networkPacketReady() ;
private:
VideoEncoder *_video_encoder ;
VideoProcessor *_video_processor ;
QTimer *_timer ;
CvCapture *_capture_device ;

80
plugins/VOIP/gui/VOIPChatWidgetHolder.cpp
View File

@ -123,8 +123,7 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n
inputAudioDevice = NULL ;
inputVideoDevice = new QVideoInputDevice(mChatWidget) ; // not started yet ;-)
inputVideoProcessor = new JPEGVideoEncoder ;
outputVideoProcessor = new JPEGVideoDecoder ;
videoProcessor = new VideoProcessor ;
// Make a widget with two video devices, one for echo, and one for the talking peer.
videoWidget = new QWidget(mChatWidget) ;
@ -144,8 +143,8 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n
mChatWidget->addChatHorizontalWidget(videoWidget) ;
inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ;
inputVideoDevice->setVideoEncoder(inputVideoProcessor) ;
outputVideoProcessor->setDisplayTarget(outputVideoDevice) ;
inputVideoDevice->setVideoProcessor(videoProcessor) ;
videoProcessor->setDisplayTarget(outputVideoDevice) ;
}
VOIPChatWidgetHolder::~VOIPChatWidgetHolder()
@ -154,8 +153,7 @@ VOIPChatWidgetHolder::~VOIPChatWidgetHolder()
inputAudioDevice->stop() ;
delete inputVideoDevice ;
delete inputVideoProcessor ;
delete outputVideoProcessor ;
delete videoProcessor ;
button_map::iterator it = buttonMapTakeVideo.begin();
while (it != buttonMapTakeVideo.end()) {
@ -287,42 +285,50 @@ void VOIPChatWidgetHolder::toggleVideoCapture()
void VOIPChatWidgetHolder::addVideoData(const RsPeerId &peer_id, QByteArray* array)
{
if (!videoCaptureToggleButton->isChecked()) {
if (mChatWidget) {
QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str());
if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId
button_map::iterator it = buttonMapTakeVideo.find(buttonName);
if (it == buttonMapTakeVideo.end()){
mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime()
, tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM);
RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call"));
button->setToolTip(tr("Activate camera"));
button->setStyleSheet(QString("border: 1px solid #199909;")
.append("font-size: 12pt; color: white;")
.append("min-width: 128px; min-height: 24px;")
.append("border-radius: 6px;")
.append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, "
"stop: 0 #22c70d, stop: 1 #116a06);")
if (!videoCaptureToggleButton->isChecked())
{
if (mChatWidget) {
QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str());
if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId
button_map::iterator it = buttonMapTakeVideo.find(buttonName);
if (it == buttonMapTakeVideo.end()){
mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime()
, tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM);
RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call"));
button->setToolTip(tr("Activate camera"));
button->setStyleSheet(QString("border: 1px solid #199909;")
.append("font-size: 12pt; color: white;")
.append("min-width: 128px; min-height: 24px;")
.append("border-radius: 6px;")
.append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, "
"stop: 0 #22c70d, stop: 1 #116a06);")
);
);
button->updateImage();
button->updateImage();
connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture()));
connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter()));
connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave()));
connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture()));
connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter()));
connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave()));
buttonMapTakeVideo.insert(buttonName, button);
}
}
buttonMapTakeVideo.insert(buttonName, button);
}
}
//TODO make a sound for the incoming call
// soundManager->play(VOIP_SOUND_INCOMING_CALL);
if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id);
//TODO make a sound for the incoming call
// soundManager->play(VOIP_SOUND_INCOMING_CALL);
if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id);
} else {
outputVideoProcessor->receiveEncodedData((unsigned char *)array->data(),array->size()) ;
}
}
else
{
RsVOIPDataChunk chunk ;
chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
chunk.size = array->size() ;
chunk.data = array->data() ;
videoProcessor->receiveEncodedData(chunk) ;
}
}
void VOIPChatWidgetHolder::botMouseEnter()
@ -359,7 +365,7 @@ void VOIPChatWidgetHolder::botMouseLeave()
void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec)
{
inputVideoProcessor->setMaximumFrameRate(bytes_per_sec) ;
videoProcessor->setMaximumBandwidth(bytes_per_sec) ;
}
void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array)

6
plugins/VOIP/gui/VOIPChatWidgetHolder.h
View File

@ -34,8 +34,7 @@ class QAudioInput;
class QAudioOutput;
class QVideoInputDevice ;
class QVideoOutputDevice ;
class VideoEncoder ;
class VideoDecoder ;
class VideoProcessor ;
#define VOIP_SOUND_INCOMING_CALL "VOIP_incoming_call"
@ -82,8 +81,7 @@ protected:
QWidget *videoWidget ; // pointer to call show/hide
VideoEncoder *inputVideoProcessor;
VideoDecoder *outputVideoProcessor;
VideoProcessor *videoProcessor;
// Additional buttons to the chat bar
QToolButton *audioListenToggleButton ;

686
plugins/VOIP/gui/VideoProcessor.cpp
View File

@ -1,4 +1,6 @@
#include <iostream>
#include <assert.h>
#include <malloc.h>
#include <QByteArray>
#include <QBuffer>
@ -7,67 +9,673 @@
#include "VideoProcessor.h"
#include "QVideoDevice.h"
VideoDecoder::VideoDecoder()
#include <math.h>
extern "C" {
#include <libavcodec/avcodec.h>
#include <libavutil/opt.h>
#include <libavutil/channel_layout.h>
#include <libavutil/common.h>
#include <libavutil/imgutils.h>
#include <libavutil/mathematics.h>
#include <libavutil/samplefmt.h>
}
//#define DEBUG_MPEG_VIDEO 1
VideoProcessor::VideoProcessor()
:_encoded_frame_size(640,480) , vpMtx("VideoProcessor")
{
_output_device = NULL ;
_decoded_output_device = NULL ;
//_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO;
_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO;
_estimated_bandwidth_in = 0 ;
_estimated_bandwidth_out = 0 ;
_target_bandwidth_out = 30*1024 ; // 30 KB/s
_total_encoded_size_in = 0 ;
_total_encoded_size_out = 0 ;
_last_bw_estimate_in_TS = time(NULL) ;
_last_bw_estimate_out_TS = time(NULL) ;
}
bool VideoEncoder::addImage(const QImage& img)
VideoProcessor::~VideoProcessor()
{
encodeData(img) ;
return true ;
// clear encoding queue
RS_STACK_MUTEX(vpMtx) ;
while(!_encoded_out_queue.empty())
{
_encoded_out_queue.back().clear() ;
_encoded_out_queue.pop_back() ;
}
}
bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk)
bool VideoProcessor::processImage(const QImage& img)
{
if(_out_queue.empty())
VideoCodec *codec ;
switch(_encoding_current_codec)
{
case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ;
break ;
case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ;
break ;
default:
codec = NULL ;
}
// std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl;
if(codec)
{
RsVOIPDataChunk chunk ;
if(codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),_target_bandwidth_out,chunk) && chunk.size > 0)
{
RS_STACK_MUTEX(vpMtx) ;
_encoded_out_queue.push_back(chunk) ;
_total_encoded_size_out += chunk.size ;
}
time_t now = time(NULL) ;
if(now > _last_bw_estimate_out_TS)
{
RS_STACK_MUTEX(vpMtx) ;
_estimated_bandwidth_out = uint32_t(0.75*_estimated_bandwidth_out + 0.25 * (_total_encoded_size_out / (float)(now - _last_bw_estimate_out_TS))) ;
_total_encoded_size_out = 0 ;
_last_bw_estimate_out_TS = now ;
#ifdef DEBUG_VIDEO_OUTPUT_DEVICE
std::cerr << "new bw estimate: " << _estimated_bw << std::endl;
#endif
}
return true ;
}
else
{
std::cerr << "No codec for codec ID = " << _encoding_current_codec << ". Please call VideoProcessor::setCurrentCodec()" << std::endl;
return false ;
}
}
bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk)
{
RS_STACK_MUTEX(vpMtx) ;
if(_encoded_out_queue.empty())
return false ;
chunk = _out_queue.front() ;
_out_queue.pop_front() ;
chunk = _encoded_out_queue.front() ;
_encoded_out_queue.pop_front() ;
return true ;
}
void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size)
void VideoProcessor::setInternalFrameSize(QSize s)
{
_output_device->showFrame(decodeData(data,size)) ;
_encoded_frame_size = s ;
}
QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint32_t size)
void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk)
{
QByteArray qb((char*)encoded_image_data,size) ;
QImage image ;
if(image.loadFromData(qb,"JPEG"))
return image ;
static const int HEADER_SIZE = 4 ;
// read frame type. Use first 4 bytes to give info about content.
//
// Byte Meaning Values
// 00 Codec CODEC_ID_JPEG_VIDEO Basic Jpeg codec
// CODEC_ID_DDWT_VIDEO Differential wavelet compression
//
// 01 Unused Might be useful later
//
// 0203 Flags Codec specific flags.
//
if(chunk.size < HEADER_SIZE)
{
std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << chunk.size << std::endl;
return ;
}
uint32_t codid = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ;
//uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ;
VideoCodec *codec ;
switch(codid)
{
case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ;
break ;
case VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO: codec = &_mpeg_video_codec ;
break ;
default:
codec = NULL ;
}
QImage img ;
if(codec == NULL)
{
std::cerr << "Unknown decoding codec: " << codid << std::endl;
return ;
}
{
RS_STACK_MUTEX(vpMtx) ;
_total_encoded_size_in += chunk.size ;
time_t now = time(NULL) ;
if(now > _last_bw_estimate_in_TS)
{
_estimated_bandwidth_in = uint32_t(0.75*_estimated_bandwidth_in + 0.25 * (_total_encoded_size_in / (float)(now - _last_bw_estimate_in_TS))) ;
_total_encoded_size_in = 0 ;
_last_bw_estimate_in_TS = now ;
#ifdef DEBUG_VIDEO_OUTPUT_DEVICE
std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl;
#endif
}
}
if(!codec->decodeData(chunk,img))
{
std::cerr << "No image decoded. Probably in the middle of something..." << std::endl;
return ;
}
if(_decoded_output_device)
_decoded_output_device->showFrame(img) ;
}
void VideoProcessor::setMaximumBandwidth(uint32_t bytes_per_sec)
{
std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl;
_target_bandwidth_out = bytes_per_sec ;
}
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
//
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
JPEGVideo::JPEGVideo()
: _encoded_ref_frame_max_distance(10),_encoded_ref_frame_count(10)
{
}
bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image)
{
// now see if the frame is a differential frame, or just a reference frame.
uint16_t codec = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ;
uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ;
assert(codec == VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO) ;
// un-compress image data
QByteArray qb((char*)&((uint8_t*)chunk.data)[HEADER_SIZE],(int)chunk.size - HEADER_SIZE) ;
if(!image.loadFromData(qb,"JPEG"))
{
std::cerr << "image.loadFromData(): returned an error.: " << std::endl;
return false ;
}
if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME)
{
if(_decoded_reference_frame.size() != image.size())
{
std::cerr << "Bad reference frame!" << std::endl;
return false ;
}
QImage res = _decoded_reference_frame ;
for(int i=0;i<image.byteCount();++i)
{
int new_val = (int)res.bits()[i] + ((int)image.bits()[i] - 128) ;
res.bits()[i] = std::max(0,std::min(255,new_val)) ;
}
image = res ;
}
else
_decoded_reference_frame = image ;
return true ;
}
bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDataChunk& voip_chunk)
{
// check if we make a diff image, or if we use the full frame.
QImage encoded_frame ;
bool differential_frame ;
if(_encoded_ref_frame_count++ < _encoded_ref_frame_max_distance && image.size() == _encoded_reference_frame.size())
{
// compute difference with reference frame.
encoded_frame = image ;
for(int i=0;i<image.byteCount();++i)
{
// We cannot use basic modulo 256 arithmetic, because the decompressed JPeg frames do not follow the same rules (values are clamped)
// and cause color blotches when perturbated by a differential frame.
int diff = ( (int)image.bits()[i] - (int)_encoded_reference_frame.bits()[i]) + 128;
encoded_frame.bits()[i] = (unsigned char)std::max(0,std::min(255,diff)) ;
}
differential_frame = true ;
}
else
{
_encoded_ref_frame_count = 0 ;
_encoded_reference_frame = image ;
encoded_frame = image ;
differential_frame = false ;
}
QByteArray qb ;
QBuffer buffer(&qb) ;
buffer.open(QIODevice::WriteOnly) ;
encoded_frame.save(&buffer,"JPEG") ;
voip_chunk.data = malloc(HEADER_SIZE + qb.size());
// build header
uint32_t flags = differential_frame ? JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME : 0x0 ;
((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO & 0xff ;
((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO >> 8) & 0xff ;
((unsigned char *)voip_chunk.data)[2] = flags & 0xff ;
((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ;
memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],qb.data(),qb.size()) ;
voip_chunk.size = HEADER_SIZE + qb.size() ;
voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
return true ;
}
FFmpegVideo::FFmpegVideo()
{
// Encoding
encoding_codec = NULL ;
encoding_frame_buffer = NULL ;
encoding_context = NULL ;
//AVCodecID codec_id = AV_CODEC_ID_H264 ;
//AVCodecID codec_id = AV_CODEC_ID_MPEG2VIDEO;
AVCodecID codec_id = AV_CODEC_ID_MPEG4;
uint8_t endcode[] = { 0, 0, 1, 0xb7 };
/* find the mpeg1 video encoder */
encoding_codec = avcodec_find_encoder(codec_id);
if (!encoding_codec) throw("AV codec not found for codec id ") ;
encoding_context = avcodec_alloc_context3(encoding_codec);
if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context");
/* put sample parameters */
encoding_context->bit_rate = 10*1024 ; // default bitrate is 30KB/s
encoding_context->bit_rate_tolerance = encoding_context->bit_rate ;
#ifdef USE_VARIABLE_BITRATE
encoding_context->rc_min_rate = 0;
encoding_context->rc_max_rate = 10*1024;//encoding_context->bit_rate;
encoding_context->rc_buffer_size = 10*1024*1024;
encoding_context->rc_initial_buffer_occupancy = (int) ( 0.9 * encoding_context->rc_buffer_size);
encoding_context->rc_max_available_vbv_use = 1.0;
encoding_context->rc_min_vbv_overflow_use = 0.0;
#else
encoding_context->rc_min_rate = 0;
encoding_context->rc_max_rate = 0;
encoding_context->rc_buffer_size = 0;
#endif
encoding_context->flags |= CODEC_FLAG_PSNR;
encoding_context->flags |= CODEC_FLAG_TRUNCATED;
encoding_context->flags |= CODEC_CAP_PARAM_CHANGE;
encoding_context->i_quant_factor = 0.769f;
encoding_context->b_quant_factor = 1.4f;
encoding_context->time_base.num = 1;
encoding_context->time_base.den = 15;//framesPerSecond;
encoding_context->qmin = 1;
encoding_context->qmax = 51;
encoding_context->max_qdiff = 4;
//encoding_context->me_method = ME_HEX;
//encoding_context->max_b_frames = 4;
//encoding_context->flags |= CODEC_FLAG_LOW_DELAY; // MPEG2 only
//encoding_context->partitions = X264_PART_I4X4 | X264_PART_I8X8 | X264_PART_P8X8 | X264_PART_P4X4 | X264_PART_B8X8;
//encoding_context->crf = 0.0f;
//encoding_context->cqp = 26;
/* resolution must be a multiple of two */
encoding_context->width = 640;//176;
encoding_context->height = 480;//144;
/* frames per second */
encoding_context->time_base = (AVRational){1,25};
/* emit one intra frame every ten frames
* check frame pict_type before passing frame
* to encoder, if frame->pict_type is AV_PICTURE_TYPE_I
* then gop_size is ignored and the output of encoder
* will always be I frame irrespective to gop_size
*/
encoding_context->gop_size = 100;
//encoding_context->max_b_frames = 1;
encoding_context->pix_fmt = AV_PIX_FMT_YUV420P; //context->pix_fmt = AV_PIX_FMT_RGB24;
if (codec_id == AV_CODEC_ID_H264)
av_opt_set(encoding_context->priv_data, "preset", "slow", 0);
/* open it */
if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0)
throw std::runtime_error( "AV: Could not open codec context. Something's wrong.");
encoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ;
if(!encoding_frame_buffer)
throw std::runtime_error("AV: could not allocate frame buffer.") ;
encoding_frame_buffer->format = encoding_context->pix_fmt;
encoding_frame_buffer->width = encoding_context->width;
encoding_frame_buffer->height = encoding_context->height;
/* the image can be allocated by any means and av_image_alloc() is
* just the most convenient way if av_malloc() is to be used */
int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize,
encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32);
if (ret < 0)
throw std::runtime_error("AV: Could not allocate raw picture buffer");
encoding_frame_count = 0 ;
// Decoding
decoding_codec = avcodec_find_decoder(codec_id);
if (!decoding_codec)
throw("AV codec not found for codec id ") ;
decoding_context = avcodec_alloc_context3(decoding_codec);
if(!decoding_context)
throw std::runtime_error("AV: Could not allocate video codec decoding context");
decoding_context->width = encoding_context->width;
decoding_context->height = encoding_context->height;
decoding_context->pix_fmt = AV_PIX_FMT_YUV420P;
if(decoding_codec->capabilities & CODEC_CAP_TRUNCATED)
decoding_context->flags |= CODEC_FLAG_TRUNCATED; // we do not send complete frames
if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0)
throw("AV codec open action failed! ") ;
decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ;
av_init_packet(&decoding_buffer);
decoding_buffer.data = NULL ;
decoding_buffer.size = 0 ;
//ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32);
//if (ret < 0)
//throw std::runtime_error("AV: Could not allocate raw picture buffer");
// debug
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Dumping captured data to file tmpvideo.mpg" << std::endl;
encoding_debug_file = fopen("tmpvideo.mpg","w") ;
#endif
}
FFmpegVideo::~FFmpegVideo()
{
avcodec_close(encoding_context);
avcodec_close(decoding_context);
av_free(encoding_context);
av_free(decoding_context);
av_freep(&encoding_frame_buffer->data[0]);
av_freep(&decoding_frame_buffer->data[0]);
free(encoding_frame_buffer);
free(decoding_frame_buffer);
}
#define MAX_FFMPEG_ENCODING_BITRATE 81920
bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk)
{
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : ";
#endif
QImage input ;
if(target_encoding_bitrate > MAX_FFMPEG_ENCODING_BITRATE)
{
std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl;
target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ;
}
//encoding_context->bit_rate = target_encoding_bitrate;
encoding_context->rc_max_rate = target_encoding_bitrate;
//encoding_context->bit_rate_tolerance = target_encoding_bitrate;
if(image.width() != encoding_frame_buffer->width || image.height() != encoding_frame_buffer->height)
input = image.scaled(QSize(encoding_frame_buffer->width,encoding_frame_buffer->height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ;
else
input = image ;
/* prepare a dummy image */
/* Y */
for (int y = 0; y < encoding_context->height/2; y++)
for (int x = 0; x < encoding_context->width/2; x++)
{
QRgb pix00 = input.pixel(QPoint(2*x+0,2*y+0)) ;
QRgb pix01 = input.pixel(QPoint(2*x+0,2*y+1)) ;
QRgb pix10 = input.pixel(QPoint(2*x+1,2*y+0)) ;
QRgb pix11 = input.pixel(QPoint(2*x+1,2*y+1)) ;
int R00 = (pix00 >> 16) & 0xff ; int G00 = (pix00 >> 8) & 0xff ; int B00 = (pix00 >> 0) & 0xff ;
int R01 = (pix01 >> 16) & 0xff ; int G01 = (pix01 >> 8) & 0xff ; int B01 = (pix01 >> 0) & 0xff ;
int R10 = (pix10 >> 16) & 0xff ; int G10 = (pix10 >> 8) & 0xff ; int B10 = (pix10 >> 0) & 0xff ;
int R11 = (pix11 >> 16) & 0xff ; int G11 = (pix11 >> 8) & 0xff ; int B11 = (pix11 >> 0) & 0xff ;
int Y00 = (0.257 * R00) + (0.504 * G00) + (0.098 * B00) + 16 ;
int Y01 = (0.257 * R01) + (0.504 * G01) + (0.098 * B01) + 16 ;
int Y10 = (0.257 * R10) + (0.504 * G10) + (0.098 * B10) + 16 ;
int Y11 = (0.257 * R11) + (0.504 * G11) + (0.098 * B11) + 16 ;
float R = 0.25*(R00+R01+R10+R11) ;
float G = 0.25*(G00+G01+G10+G11) ;
float B = 0.25*(B00+B01+B10+B11) ;
int U = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ;
int V = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ;
encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y00)); // Y
encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y01)); // Y
encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y10)); // Y
encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y11)); // Y
encoding_frame_buffer->data[1][y * encoding_frame_buffer->linesize[1] + x] = std::min(255,std::max(0,U));// Cr
encoding_frame_buffer->data[2][y * encoding_frame_buffer->linesize[2] + x] = std::min(255,std::max(0,V));// Cb
}
encoding_frame_buffer->pts = encoding_frame_count++;
/* encode the image */
int got_output = 0;
AVFrame *frame = encoding_frame_buffer ;
AVPacket pkt ;
av_init_packet(&pkt);
pkt.data = NULL; // packet data will be allocated by the encoder
pkt.size = 0;
// do
// {
int ret = avcodec_encode_video2(encoding_context, &pkt, frame, &got_output) ;
if (ret < 0)
{
std::cerr << "Error encoding frame!" << std::endl;
return false ;
}
// frame = NULL ; // next attempts: do not encode anything. Do this to just flush the buffer
//
// } while(got_output) ;
if(got_output)
{
voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ;
uint32_t flags = 0;
((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ;
((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO >> 8) & 0xff ;
((unsigned char *)voip_chunk.data)[2] = flags & 0xff ;
((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ;
memcpy(&((unsigned char*)voip_chunk.data)[HEADER_SIZE],pkt.data,pkt.size) ;
voip_chunk.size = pkt.size + HEADER_SIZE;
voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Output : " << pkt.size << " bytes." << std::endl;
fwrite(pkt.data,1,pkt.size,encoding_debug_file) ;
fflush(encoding_debug_file) ;
#endif
av_free_packet(&pkt);
return true ;
}
else
{
std::cerr << "image.loadFromData(): returned an error.: " << std::endl;
return QImage() ;
voip_chunk.data = NULL;
voip_chunk.size = 0;
voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
std::cerr << "No output produced." << std::endl;
return false ;
}
}
void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec)
bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image)
{
std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl;
}
void JPEGVideoEncoder::encodeData(const QImage& image)
{
QByteArray qb ;
QBuffer buffer(&qb) ;
buffer.open(QIODevice::WriteOnly) ;
image.save(&buffer,"JPEG") ;
RsVOIPDataChunk voip_chunk ;
voip_chunk.data = malloc(qb.size());
memcpy(voip_chunk.data,qb.data(),qb.size()) ;
voip_chunk.size = qb.size() ;
voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
_out_queue.push_back(voip_chunk) ;
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Decoding data of size " << chunk.size << std::endl;
#endif
unsigned char *buff ;
if(decoding_buffer.data != NULL)
{
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Completing buffer with size " << chunk.size - HEADER_SIZE + decoding_buffer.size << ": copying existing "
<< decoding_buffer.size << " bytes. Adding new " << chunk.size - HEADER_SIZE<< " bytes " << std::endl;
#endif
uint32_t s = chunk.size - HEADER_SIZE + decoding_buffer.size ;
unsigned char *tmp = (unsigned char*)memalign(16,s + FF_INPUT_BUFFER_PADDING_SIZE) ;
memset(tmp,0,s+FF_INPUT_BUFFER_PADDING_SIZE) ;
memcpy(tmp,decoding_buffer.data,decoding_buffer.size) ;
free(decoding_buffer.data) ;
buff = &tmp[decoding_buffer.size] ;
decoding_buffer.size = s ;
decoding_buffer.data = tmp ;
}
else
{
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl;
#endif
decoding_buffer.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ;
decoding_buffer.size = chunk.size - HEADER_SIZE ;
memset(decoding_buffer.data,0,decoding_buffer.size + FF_INPUT_BUFFER_PADDING_SIZE) ;
buff = decoding_buffer.data ;
}
memcpy(buff,&((unsigned char*)chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ;
int got_frame = 0 ;
int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ;
if(len < 0)
{
std::cerr << "Error decodign frame!" << std::endl;
return false ;
}
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Used " << len << " bytes out of " << decoding_buffer.size << ". got_frame = " << got_frame << std::endl;
#endif
if(got_frame)
{
image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ;
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl;
#endif
for (int y = 0; y < decoding_frame_buffer->height; y++)
for (int x = 0; x < decoding_frame_buffer->width; x++)
{
int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ;
int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ;
int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ;
int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ;
int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ;
int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ;
image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ;
}
}
if(len == decoding_buffer.size)
{
free(decoding_buffer.data) ;
decoding_buffer.data = NULL;
decoding_buffer.size = 0;
}
else if(len != 0)
{
#ifdef DEBUG_MPEG_VIDEO
std::cerr << "Moving remaining data (" << decoding_buffer.size - len << " bytes) back to 0" << std::endl;
#endif
memmove(decoding_buffer.data,decoding_buffer.data+len,decoding_buffer.size - len) ;
decoding_buffer.size -= len ;
}
return true ;
}

196
plugins/VOIP/gui/VideoProcessor.h
View File

@ -4,87 +4,149 @@
#include <QImage>
#include "interface/rsVOIP.h"
extern "C" {
#include <libavcodec/avcodec.h>
}
class QVideoOutputDevice ;
// This class decodes video from a stream. It keeps a queue of
// decoded frame that needs to be retrieved using the getNextImage() method.
//
class VideoDecoder
class VideoCodec
{
public:
VideoDecoder() ;
virtual ~VideoDecoder() {}
// Gets the next image to be displayed. Once returned, the image should
// be cleared from the incoming queue.
//
void setDisplayTarget(QVideoOutputDevice *odev) { _output_device = odev ; }
virtual void receiveEncodedData(const unsigned char *data,uint32_t size) ;
// returns the current (measured) frame rate in bytes per second.
//
uint32_t currentFrameRate() const;
private:
QVideoOutputDevice *_output_device ;
std::list<QImage> _image_queue ;
// Incoming data is processed by a video codec and converted into images.
//
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) = 0 ;
// // This buffer accumulated incoming encoded data, until a full packet is obtained,
// // since the stream might not send images at once. When incoming images are decoded, the
// // data is removed from the buffer.
// //
// unsigned char *buffer ;
// uint32_t buffer_size ;
};
// This class encodes video using a video codec (possibly homemade, or based on existing codecs)
// and produces a data stream that is sent to the network transfer service (e.g. p3VOIP).
//
class VideoEncoder
{
public:
VideoEncoder() {}
virtual ~VideoEncoder() {}
// Takes the next image to be encoded.
//
bool addImage(const QImage& Image) ;
bool packetReady() const { return !_out_queue.empty() ; }
bool nextPacket(RsVOIPDataChunk& ) ;
// Used to tweak the compression ratio so that the video can stream ok.
//
void setMaximumFrameRate(uint32_t bytes_per_second) ;
protected:
//virtual bool sendEncodedData(unsigned char *mem,uint32_t size) = 0 ;
virtual void encodeData(const QImage& image) = 0 ;
std::list<RsVOIPDataChunk> _out_queue ;
public:
virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) = 0;
virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) = 0;
protected:
static const uint32_t HEADER_SIZE = 0x04 ;
};
// Now derive various image encoding/decoding algorithms.
//
class JPEGVideoDecoder: public VideoDecoder
class JPEGVideo: public VideoCodec
{
protected:
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ;
public:
JPEGVideo() ;
protected:
virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ;
virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ;
static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ;
private:
QImage _decoded_reference_frame ;
QImage _encoded_reference_frame ;
uint32_t _encoded_ref_frame_max_distance ; // max distance between two reference frames.
uint32_t _encoded_ref_frame_count ;
};
class JPEGVideoEncoder: public VideoEncoder
struct AVCodec ;
struct AVCodecContext ;
struct AVFrame ;
struct AVPacket ;
class FFmpegVideo: public VideoCodec
{
public:
FFmpegVideo() ;
~FFmpegVideo() ;
protected:
virtual bool encodeData(const QImage& Image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& chunk) ;
virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ;
private:
AVCodec *encoding_codec;
AVCodec *decoding_codec;
AVCodecContext *encoding_context;
AVCodecContext *decoding_context;
AVFrame *encoding_frame_buffer ;
AVFrame *decoding_frame_buffer ;
AVPacket decoding_buffer;
uint64_t encoding_frame_count ;
FILE *encoding_debug_file ;
};
// This class decodes video from a stream. It keeps a queue of
// decoded frame that needs to be retrieved using the getNextImage() method.
//
class VideoProcessor
{
public:
JPEGVideoEncoder() {}
VideoProcessor() ;
virtual ~VideoProcessor() ;
enum CodecId {
VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000,
VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO = 0x0001,
VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002,
VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO = 0x0003
};
// =====================================================================================
// =------------------------------------ DECODING -------------------------------------=
// =====================================================================================
// Gets the next image to be displayed. Once returned, the image should
// be cleared from the incoming queue.
//
void setDisplayTarget(QVideoOutputDevice *odev) { _decoded_output_device = odev ; }
virtual void receiveEncodedData(const RsVOIPDataChunk& chunk) ;
// returns the current (measured) frame rate in bytes per second.
//
uint32_t currentBandwidthIn() const { return _estimated_bandwidth_in ; }
private:
QVideoOutputDevice *_decoded_output_device ;
std::list<QImage> _decoded_image_queue ;
// =====================================================================================
// =------------------------------------ ENCODING -------------------------------------=
// =====================================================================================
public:
// Takes the next image to be encoded.
//
bool processImage(const QImage& Image) ;
bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; }
bool nextEncodedPacket(RsVOIPDataChunk& ) ;
// Used to tweak the compression ratio so that the video can stream ok.
//
void setMaximumBandwidth(uint32_t bytes_per_second) ;
void setInternalFrameSize(QSize) ;
// returns the current encoding frame rate in bytes per second.
//
uint32_t currentBandwidthOut() const { return _estimated_bandwidth_out ; }
protected:
virtual void encodeData(const QImage& Image) ;
std::list<RsVOIPDataChunk> _encoded_out_queue ;
QSize _encoded_frame_size ;
// =====================================================================================
// =------------------------------------- Codecs --------------------------------------=
// =====================================================================================
JPEGVideo _jpeg_video_codec ;
FFmpegVideo _mpeg_video_codec ;
uint16_t _encoding_current_codec ;
time_t _last_bw_estimate_in_TS;
time_t _last_bw_estimate_out_TS;
uint32_t _total_encoded_size_in ;
uint32_t _total_encoded_size_out ;
float _estimated_bandwidth_in ;
float _estimated_bandwidth_out ;
float _target_bandwidth_out ;
RsMutex vpMtx ;
};

4
plugins/VOIP/interface/rsVOIP.h
View File

@ -51,7 +51,9 @@ class RsVOIPPongResult
struct RsVOIPDataChunk
{
typedef enum { RS_VOIP_DATA_TYPE_AUDIO, RS_VOIP_DATA_TYPE_VIDEO } RsVOIPDataType ;
typedef enum { RS_VOIP_DATA_TYPE_UNKNOWN = 0x00,
RS_VOIP_DATA_TYPE_AUDIO = 0x01,
RS_VOIP_DATA_TYPE_VIDEO = 0x02 } RsVOIPDataType ;
void *data ; // create/delete using malloc/free.
uint32_t size ;

85
plugins/VOIP/services/rsVOIPItems.cc
View File

@ -76,6 +76,19 @@ std::ostream& RsVOIPProtocolItem::print(std::ostream &out, uint16_t indent)
printRsItemEnd(out, "RsVOIPProtocolItem", indent);
return out;
}
std::ostream& RsVOIPBandwidthItem::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsVOIPBandwidthItem", indent);
uint16_t int_Indent = indent + 2;
printIndent(out, int_Indent);
out << "flags: " << std::hex << flags << std::dec << std::endl;
printIndent(out, int_Indent);
out << "speed: " << bytes_per_sec << std::endl;
printRsItemEnd(out, "RsVOIPBandwidthItem", indent);
return out;
}
std::ostream& RsVOIPDataItem::print(std::ostream &out, uint16_t indent)
{
printRsItemBase(out, "RsVOIPDataItem", indent);
@ -100,6 +113,14 @@ uint32_t RsVOIPDataItem::serial_size() const
return s;
}
uint32_t RsVOIPBandwidthItem::serial_size() const
{
uint32_t s = 8; /* header */
s += 4; /* flags */
s += 4; /* bandwidth */
return s;
}
uint32_t RsVOIPProtocolItem::serial_size() const
{
uint32_t s = 8; /* header */
@ -150,6 +171,40 @@ bool RsVOIPProtocolItem::serialise(void *data, uint32_t& pktsize)
return ok;
}
bool RsVOIPBandwidthItem::serialise(void *data, uint32_t& pktsize)
{
uint32_t tlvsize = serial_size() ;
uint32_t offset = 0;
if (pktsize < tlvsize)
return false; /* not enough space */
pktsize = tlvsize;
bool ok = true;
ok &= setRsItemHeader(data, tlvsize, PacketId(), tlvsize);
#ifdef RSSERIAL_DEBUG
std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Header: " << ok << std::endl;
std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size: " << tlvsize << std::endl;
#endif
/* skip the header */
offset += 8;
/* add mandatory parts first */
ok &= setRawUInt32(data, tlvsize, &offset, flags);
ok &= setRawUInt32(data, tlvsize, &offset, bytes_per_sec);
if (offset != tlvsize)
{
ok = false;
std::cerr << "RsVOIPSerialiser::serialiseVOIPBandwidthItem() Size Error! " << std::endl;
}
return ok;
}
/* serialise the data to the buffer */
bool RsVOIPDataItem::serialise(void *data, uint32_t& pktsize)
{
@ -254,6 +309,36 @@ RsVOIPProtocolItem::RsVOIPProtocolItem(void *data, uint32_t pktsize)
if (!ok)
throw std::runtime_error("Deserialisation error!") ;
}
RsVOIPBandwidthItem::RsVOIPBandwidthItem(void *data, uint32_t pktsize)
: RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH)
{
/* get the type and size */
uint32_t rstype = getRsItemId(data);
uint32_t rssize = getRsItemSize(data);
uint32_t offset = 0;
if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (RS_SERVICE_TYPE_VOIP_PLUGIN != getRsItemService(rstype)) || (RS_PKT_SUBTYPE_VOIP_BANDWIDTH != getRsItemSubType(rstype)))
throw std::runtime_error("Wrong packet type!") ;
if (pktsize < rssize) /* check size */
throw std::runtime_error("Not enough size!") ;
bool ok = true;
/* skip the header */
offset += 8;
/* get mandatory parts first */
ok &= getRawUInt32(data, rssize, &offset, &flags);
ok &= getRawUInt32(data, rssize, &offset, &bytes_per_sec);
if (offset != rssize)
throw std::runtime_error("Deserialisation error!") ;
if (!ok)
throw std::runtime_error("Deserialisation error!") ;
}
RsVOIPPingItem::RsVOIPPingItem(void *data, uint32_t pktsize)
: RsVOIPItem(RS_PKT_SUBTYPE_VOIP_PING)
{

29
plugins/VOIP/services/rsVOIPItems.h
View File

@ -55,11 +55,12 @@
const uint16_t RS_SERVICE_TYPE_VOIP_PLUGIN = 0xa021;
const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01;
const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02;
const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ;
// 0x04 is unused because of a change in the protocol
const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x05 ;
const uint8_t RS_PKT_SUBTYPE_VOIP_PING = 0x01;
const uint8_t RS_PKT_SUBTYPE_VOIP_PONG = 0x02;
const uint8_t RS_PKT_SUBTYPE_VOIP_PROTOCOL = 0x03 ;
// 0x04,0x05 is unused because of a change in the protocol
const uint8_t RS_PKT_SUBTYPE_VOIP_BANDWIDTH = 0x06 ;
const uint8_t RS_PKT_SUBTYPE_VOIP_DATA = 0x07 ;
const uint8_t QOS_PRIORITY_RS_VOIP = 9 ;
@ -117,9 +118,27 @@ class RsVOIPDataItem: public RsVOIPItem
uint32_t flags ;
uint32_t data_size ;
void *voip_data ;
};
class RsVOIPBandwidthItem: public RsVOIPItem
{
public:
RsVOIPBandwidthItem() :RsVOIPItem(RS_PKT_SUBTYPE_VOIP_BANDWIDTH) {}
RsVOIPBandwidthItem(void *data,uint32_t size) ; // de-serialization
virtual bool serialise(void *data,uint32_t& size) ;
virtual uint32_t serial_size() const ;
virtual ~RsVOIPBandwidthItem() {}
virtual std::ostream& print(std::ostream &out, uint16_t indent = 0);
uint32_t flags ; // is that incoming or expected bandwidth?
uint32_t bytes_per_sec ; // bandwidth in bytes per sec.
};
class RsVOIPProtocolItem: public RsVOIPItem
{
public: