Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-06-17 19:09:29 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactored video processing classes, to allow multiple codecs. Not working yet.

Browse source
This commit is contained in:
csoler 2015-08-14 16:44:20 -04:00
parent 3bda2c2660
commit cd97fd9682
8 changed files with 284 additions and 237 deletions
Download patch file Download diff file Expand all files Collapse all files

50
plugins/VOIP/gui/AudioInputConfig.cpp
View file

@ -34,6 +34,7 @@
#include "AudioInputConfig.h" #include "AudioInputConfig.h"
#include "audiodevicehelper.h" #include "audiodevicehelper.h"
#include "AudioWizard.h" #include "AudioWizard.h"
#include "gui/VideoProcessor.h"
#include "gui/common/RSGraphWidget.h" #include "gui/common/RSGraphWidget.h"
#include "util/RsProtectedTimer.h" #include "util/RsProtectedTimer.h"
@ -99,35 +100,36 @@ voipGraph::voipGraph(QWidget *parent)
AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags) AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags)
: ConfigPage(parent, flags) : ConfigPage(parent, flags)
{ {
std::cerr << "Creating audioInputConfig object" << std::endl; std::cerr << "Creating audioInputConfig object" << std::endl;
/* Invoke the Qt Designer generated object setup routine */ /* Invoke the Qt Designer generated object setup routine */
ui.setupUi(this); ui.setupUi(this);
loaded = false; loaded = false;
inputAudioProcessor = NULL; inputAudioProcessor = NULL;
inputAudioDevice = NULL; inputAudioDevice = NULL;
abSpeech = NULL; abSpeech = NULL;
qtTick = NULL; qtTick = NULL;
// Create the video pipeline. // Create the video pipeline.
// //
videoInput = new QVideoInputDevice(this) ; videoInput = new QVideoInputDevice(this) ;
videoInput->setEchoVideoTarget(ui.videoDisplay) ; videoInput->setEchoVideoTarget(ui.videoDisplay) ;
videoInput->setVideoEncoder(new JPEGVideoEncoder()) ;
videoDecoder = new JPEGVideoDecoder; videoProcessor = new VideoProcessor() ;
videoDecoder->setDisplayTarget(NULL) ; videoProcessor->setDisplayTarget(NULL) ;
graph_source = new voipGraphSource ; videoInput->setVideoProcessor(videoProcessor) ;
ui.voipBwGraph->setSource(graph_source);
graph_source->setVideoInput(videoInput) ; graph_source = new voipGraphSource ;
graph_source->setCollectionTimeLimit(1000*300) ; ui.voipBwGraph->setSource(graph_source);
graph_source->start() ;
QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ; graph_source->setVideoInput(videoInput) ;
graph_source->setCollectionTimeLimit(1000*300) ;
graph_source->start() ;
QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ;
} }
void AudioInputConfig::togglePreview(bool b) void AudioInputConfig::togglePreview(bool b)
@ -135,12 +137,12 @@ void AudioInputConfig::togglePreview(bool b)
if(b) if(b)
{ {
videoInput->setEchoVideoTarget(NULL) ; videoInput->setEchoVideoTarget(NULL) ;
videoDecoder->setDisplayTarget(ui.videoDisplay) ; videoProcessor->setDisplayTarget(ui.videoDisplay) ;
} }
else else
{ {
videoInput->setEchoVideoTarget(ui.videoDisplay) ; videoInput->setEchoVideoTarget(ui.videoDisplay) ;
videoDecoder->setDisplayTarget(NULL) ; videoProcessor->setDisplayTarget(NULL) ;
} }
} }
@ -367,7 +369,7 @@ void AudioInputConfig::on_Tick_timeout() {
while(videoInput->getNextEncodedPacket(chunk)) while(videoInput->getNextEncodedPacket(chunk))
{ {
videoDecoder->receiveEncodedData(static_cast<const unsigned char*>(chunk.data),chunk.size) ; videoProcessor->receiveEncodedData(chunk) ;
chunk.clear() ; chunk.clear() ;
} }
} }

2
plugins/VOIP/gui/AudioInputConfig.h
View file

@ -69,7 +69,7 @@ class AudioInputConfig : public ConfigPage
//VideoDecoder *videoDecoder ; //VideoDecoder *videoDecoder ;
//VideoEncoder *videoEncoder ; //VideoEncoder *videoEncoder ;
QVideoInputDevice *videoInput ; QVideoInputDevice *videoInput ;
VideoDecoder *videoDecoder ; VideoProcessor *videoProcessor ;
bool loaded; bool loaded;
voipGraphSource *graph_source ; voipGraphSource *graph_source ;

10
plugins/VOIP/gui/QVideoDevice.cpp
View file

@ -11,7 +11,7 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent)
{ {
_timer = NULL ; _timer = NULL ;
_capture_device = NULL ; _capture_device = NULL ;
_video_encoder = NULL ; _video_processor = NULL ;
_echo_output_device = NULL ; _echo_output_device = NULL ;
_estimated_bw = 0 ; _estimated_bw = 0 ;
_total_encoded_size = 0 ; _total_encoded_size = 0 ;
@ -78,11 +78,11 @@ void QVideoInputDevice::grabFrame()
QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888); QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888);
if(_video_encoder != NULL) if(_video_processor != NULL)
{ {
uint32_t encoded_size ; uint32_t encoded_size ;
_video_encoder->addImage(image,0,encoded_size) ; _video_processor->processImage(image,0,encoded_size) ;
std::cerr << "Encoded size = " << encoded_size << std::endl; std::cerr << "Encoded size = " << encoded_size << std::endl;
_total_encoded_size += encoded_size ; _total_encoded_size += encoded_size ;
@ -107,8 +107,8 @@ void QVideoInputDevice::grabFrame()
bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk)
{ {
if(_video_encoder) if(_video_processor)
return _video_encoder->nextPacket(chunk) ; return _video_processor->nextEncodedPacket(chunk) ;
else else
return false ; return false ;
} }

5
plugins/VOIP/gui/QVideoDevice.h
View file

@ -2,6 +2,7 @@
#include <QLabel> #include <QLabel>
#include "interface/rsVOIP.h" #include "interface/rsVOIP.h"
#include "gui/VideoProcessor.h"
class VideoEncoder ; class VideoEncoder ;
class CvCapture ; class CvCapture ;
@ -31,7 +32,7 @@ class QVideoInputDevice: public QObject
// Captured images are sent to this encoder. Can be NULL. // Captured images are sent to this encoder. Can be NULL.
// //
void setVideoEncoder(VideoEncoder *venc) { _video_encoder = venc ; } void setVideoProcessor(VideoProcessor *venc) { _video_processor = venc ; }
// All images received will be echoed to this target. We could use signal/slots, but it's // All images received will be echoed to this target. We could use signal/slots, but it's
// probably faster this way. Can be NULL. // probably faster this way. Can be NULL.
@ -58,7 +59,7 @@ class QVideoInputDevice: public QObject
void networkPacketReady() ; void networkPacketReady() ;
private: private:
VideoEncoder *_video_encoder ; VideoProcessor *_video_processor ;
QTimer *_timer ; QTimer *_timer ;
CvCapture *_capture_device ; CvCapture *_capture_device ;

80
plugins/VOIP/gui/VOIPChatWidgetHolder.cpp
View file

@ -123,8 +123,7 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n
inputAudioDevice = NULL ; inputAudioDevice = NULL ;
inputVideoDevice = new QVideoInputDevice(mChatWidget) ; // not started yet ;-) inputVideoDevice = new QVideoInputDevice(mChatWidget) ; // not started yet ;-)
inputVideoProcessor = new JPEGVideoEncoder ; videoProcessor = new VideoProcessor ;
outputVideoProcessor = new JPEGVideoDecoder ;
// Make a widget with two video devices, one for echo, and one for the talking peer. // Make a widget with two video devices, one for echo, and one for the talking peer.
videoWidget = new QWidget(mChatWidget) ; videoWidget = new QWidget(mChatWidget) ;
@ -144,8 +143,8 @@ VOIPChatWidgetHolder::VOIPChatWidgetHolder(ChatWidget *chatWidget, VOIPNotify *n
mChatWidget->addChatHorizontalWidget(videoWidget) ; mChatWidget->addChatHorizontalWidget(videoWidget) ;
inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ; inputVideoDevice->setEchoVideoTarget(echoVideoDevice) ;
inputVideoDevice->setVideoEncoder(inputVideoProcessor) ; inputVideoDevice->setVideoProcessor(videoProcessor) ;
outputVideoProcessor->setDisplayTarget(outputVideoDevice) ; videoProcessor->setDisplayTarget(outputVideoDevice) ;
} }
VOIPChatWidgetHolder::~VOIPChatWidgetHolder() VOIPChatWidgetHolder::~VOIPChatWidgetHolder()
@ -154,8 +153,7 @@ VOIPChatWidgetHolder::~VOIPChatWidgetHolder()
inputAudioDevice->stop() ; inputAudioDevice->stop() ;
delete inputVideoDevice ; delete inputVideoDevice ;
delete inputVideoProcessor ; delete videoProcessor ;
delete outputVideoProcessor ;
button_map::iterator it = buttonMapTakeVideo.begin(); button_map::iterator it = buttonMapTakeVideo.begin();
while (it != buttonMapTakeVideo.end()) { while (it != buttonMapTakeVideo.end()) {
@ -287,42 +285,50 @@ void VOIPChatWidgetHolder::toggleVideoCapture()
void VOIPChatWidgetHolder::addVideoData(const RsPeerId &peer_id, QByteArray* array) void VOIPChatWidgetHolder::addVideoData(const RsPeerId &peer_id, QByteArray* array)
{ {
if (!videoCaptureToggleButton->isChecked()) { if (!videoCaptureToggleButton->isChecked())
if (mChatWidget) { {
QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str()); if (mChatWidget) {
if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId QString buttonName = QString::fromUtf8(rsPeers->getPeerName(peer_id).c_str());
button_map::iterator it = buttonMapTakeVideo.find(buttonName); if (buttonName.isEmpty()) buttonName = "VoIP";//TODO maybe change all with GxsId
if (it == buttonMapTakeVideo.end()){ button_map::iterator it = buttonMapTakeVideo.find(buttonName);
mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime() if (it == buttonMapTakeVideo.end()){
, tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM); mChatWidget->addChatMsg(true, tr("VoIP Status"), QDateTime::currentDateTime(), QDateTime::currentDateTime()
RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call")); , tr("%1 inviting you to start a video conversation. do you want Accept or Decline the invitation?").arg(buttonName), ChatWidget::MSGTYPE_SYSTEM);
button->setToolTip(tr("Activate camera")); RSButtonOnText *button = mChatWidget->getNewButtonOnTextBrowser(tr("Accept Video Call"));
button->setStyleSheet(QString("border: 1px solid #199909;") button->setToolTip(tr("Activate camera"));
.append("font-size: 12pt; color: white;") button->setStyleSheet(QString("border: 1px solid #199909;")
.append("min-width: 128px; min-height: 24px;") .append("font-size: 12pt; color: white;")
.append("border-radius: 6px;") .append("min-width: 128px; min-height: 24px;")
.append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, " .append("border-radius: 6px;")
"stop: 0 #22c70d, stop: 1 #116a06);") .append("background-color: qlineargradient(x1: 0, y1: 0, x2: 0, y2: 0.67, "
"stop: 0 #22c70d, stop: 1 #116a06);")
); );
button->updateImage(); button->updateImage();
connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture())); connect(button,SIGNAL(clicked()),this,SLOT(startVideoCapture()));
connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter())); connect(button,SIGNAL(mouseEnter()),this,SLOT(botMouseEnter()));
connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave())); connect(button,SIGNAL(mouseLeave()),this,SLOT(botMouseLeave()));
buttonMapTakeVideo.insert(buttonName, button); buttonMapTakeVideo.insert(buttonName, button);
} }
} }
//TODO make a sound for the incoming call //TODO make a sound for the incoming call
// soundManager->play(VOIP_SOUND_INCOMING_CALL); // soundManager->play(VOIP_SOUND_INCOMING_CALL);
if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id); if (mVOIPNotify) mVOIPNotify->notifyReceivedVoipVideoCall(peer_id);
} else { }
outputVideoProcessor->receiveEncodedData((unsigned char *)array->data(),array->size()) ; else
} {
RsVOIPDataChunk chunk ;
chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
chunk.size = array->size() ;
chunk.data = array->data() ;
videoProcessor->receiveEncodedData(chunk) ;
}
} }
void VOIPChatWidgetHolder::botMouseEnter() void VOIPChatWidgetHolder::botMouseEnter()
@ -359,7 +365,7 @@ void VOIPChatWidgetHolder::botMouseLeave()
void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec) void VOIPChatWidgetHolder::setAcceptedBandwidth(uint32_t bytes_per_sec)
{ {
inputVideoProcessor->setMaximumFrameRate(bytes_per_sec) ; videoProcessor->setMaximumFrameRate(bytes_per_sec) ;
} }
void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array) void VOIPChatWidgetHolder::addAudioData(const RsPeerId &peer_id, QByteArray* array)

6
plugins/VOIP/gui/VOIPChatWidgetHolder.h
View file

@ -34,8 +34,7 @@ class QAudioInput;
class QAudioOutput; class QAudioOutput;
class QVideoInputDevice ; class QVideoInputDevice ;
class QVideoOutputDevice ; class QVideoOutputDevice ;
class VideoEncoder ; class VideoProcessor ;
class VideoDecoder ;
#define VOIP_SOUND_INCOMING_CALL "VOIP_incoming_call" #define VOIP_SOUND_INCOMING_CALL "VOIP_incoming_call"
@ -82,8 +81,7 @@ protected:
QWidget *videoWidget ; // pointer to call show/hide QWidget *videoWidget ; // pointer to call show/hide
VideoEncoder *inputVideoProcessor; VideoProcessor *videoProcessor;
VideoDecoder *outputVideoProcessor;
// Additional buttons to the chat bar // Additional buttons to the chat bar
QToolButton *audioListenToggleButton ; QToolButton *audioListenToggleButton ;

200
plugins/VOIP/gui/VideoProcessor.cpp
View file

@ -1,4 +1,5 @@
#include <iostream> #include <iostream>
#include <assert.h>
#include <QByteArray> #include <QByteArray>
#include <QBuffer> #include <QBuffer>
@ -7,127 +8,182 @@
#include "VideoProcessor.h" #include "VideoProcessor.h"
#include "QVideoDevice.h" #include "QVideoDevice.h"
VideoDecoder::VideoDecoder() VideoProcessor::VideoProcessor()
:_encoded_frame_size(256,256)
{ {
_output_device = NULL ; _decoded_output_device = NULL ;
} }
VideoEncoder::VideoEncoder() bool VideoProcessor::processImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size)
:_frame_size(256,256)
{ {
VideoCodec *codec ;
switch(_encoding_current_codec)
{
case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ;
break ;
case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ;
break ;
default:
codec = NULL ;
}
// std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl;
void *data = NULL;
encoded_size = 0 ;
if(codec)
{
RsVOIPDataChunk chunk ;
codec->encodeData(img.scaled(_encoded_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,chunk) ;
if(chunk.size == 0) // the codec might be buffering the frame for compression reasons
return true ;
_encoded_out_queue.push_back(chunk) ;
return true ;
}
else
return false ;
} }
bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size) bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk)
{ {
// std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl; if(_encoded_out_queue.empty())
encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ;
//encodeData(img,size_hint,encoded_size) ;
return true ;
}
bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk)
{
if(_out_queue.empty())
return false ; return false ;
chunk = _out_queue.front() ; chunk = _encoded_out_queue.front() ;
_out_queue.pop_front() ; _encoded_out_queue.pop_front() ;
return true ; return true ;
} }
void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size) void VideoProcessor::setInternalFrameSize(QSize s)
{ {
if(_output_device) _encoded_frame_size = s ;
_output_device->showFrame(decodeData(data,size)) ;
} }
QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint32_t size) void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk)
{ {
static const int HEADER_SIZE = 4 ; static const int HEADER_SIZE = 4 ;
// read frame type. Use first 4 bytes to give info about content. // read frame type. Use first 4 bytes to give info about content.
//
// Byte Meaning Values
// 00 Codec CODEC_ID_JPEG_VIDEO Basic Jpeg codec
// CODEC_ID_DDWT_VIDEO Differential wavelet compression
//
// 01 Unused Might be useful later
//
// 0203 Flags Codec specific flags.
//
if(size < HEADER_SIZE) if(chunk.size < HEADER_SIZE)
{ {
std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << size << std::endl; std::cerr << "JPEGVideoDecoder::decodeData(): Too small a data packet. size=" << chunk.size << std::endl;
return QImage() ; return ;
} }
uint32_t flags = encoded_image_data[0] + (encoded_image_data[1] << 8) ; uint32_t codid = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ;
uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ;
// un-compress image data VideoCodec *codec ;
QByteArray qb((char*)&encoded_image_data[HEADER_SIZE],(int)size - HEADER_SIZE) ; switch(codid)
QImage image ;
if(!image.loadFromData(qb,"JPEG"))
{ {
std::cerr << "image.loadFromData(): returned an error.: " << std::endl; case VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO: codec = &_jpeg_video_codec ;
return QImage() ; break ;
case VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO: codec = &_ddwt_video_codec ;
break ;
default:
codec = NULL ;
} }
QImage img ;
// now see if the frame is a differential frame, or just a reference frame. if(codec != NULL)
codec->decodeData(chunk,img) ;
if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) if(_decoded_output_device)
{ _decoded_output_device->showFrame(img) ;
if(_reference_frame.size() != image.size())
{
std::cerr << "Bad reference frame!" << std::endl;
return image ;
}
QImage res = _reference_frame ;
for(uint32_t i=0;i<image.byteCount();++i)
res.bits()[i] += (image.bits()[i] - 128) & 0xff ; // it should be -128, but we're doing modulo 256 arithmetic
return res ;
}
else
{
_reference_frame = image ;
return image ;
}
} }
void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec) void VideoProcessor::setMaximumFrameRate(uint32_t bytes_per_sec)
{ {
std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl; std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl;
} }
void VideoEncoder::setInternalFrameSize(QSize s) //////////////////////////////////////////////////////////////////////////////////////////////////////////////
{ //
_frame_size = s ; //////////////////////////////////////////////////////////////////////////////////////////////////////////////
}
JPEGVideoEncoder::JPEGVideoEncoder() JPEGVideo::JPEGVideo()
: _ref_frame_max_distance(50),_ref_frame_count(50) : _encoded_ref_frame_max_distance(50),_encoded_ref_frame_count(50)
{ {
} }
void JPEGVideoEncoder::encodeData(const QImage& image,uint32_t /* size_hint */,uint32_t& encoded_size) bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image)
{
// now see if the frame is a differential frame, or just a reference frame.
uint16_t codec = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ;
uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ;
assert(codec == VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO) ;
// un-compress image data
QByteArray qb((char*)&((uint8_t*)chunk.data)[HEADER_SIZE],(int)chunk.size - HEADER_SIZE) ;
if(!image.loadFromData(qb,"JPEG"))
{
std::cerr << "image.loadFromData(): returned an error.: " << std::endl;
return false ;
}
if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME)
{
if(_decoded_reference_frame.size() != image.size())
{
std::cerr << "Bad reference frame!" << std::endl;
return false ;
}
QImage res = _decoded_reference_frame ;
for(uint32_t i=0;i<image.byteCount();++i)
res.bits()[i] += (image.bits()[i] - 128) & 0xff ; // it should be -128, but we're doing modulo 256 arithmetic
image = res ;
}
return true ;
}
bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDataChunk& voip_chunk)
{ {
// check if we make a diff image, or if we use the full frame. // check if we make a diff image, or if we use the full frame.
QImage encoded_frame ; QImage encoded_frame ;
bool differential_frame ; bool differential_frame ;
if(_ref_frame_count++ < _ref_frame_max_distance && image.size() == _reference_frame.size()) if(_encoded_ref_frame_count++ < _encoded_ref_frame_max_distance && image.size() == _encoded_reference_frame.size())
{ {
// compute difference with reference frame. // compute difference with reference frame.
encoded_frame = image ; encoded_frame = image ;
for(uint32_t i=0;i<image.byteCount();++i) for(uint32_t i=0;i<image.byteCount();++i)
encoded_frame.bits()[i] = (image.bits()[i] - _reference_frame.bits()[i]) + 128; encoded_frame.bits()[i] = (image.bits()[i] - _encoded_reference_frame.bits()[i]) + 128;
differential_frame = true ; differential_frame = true ;
} }
else else
{ {
_ref_frame_count = 0 ; _encoded_ref_frame_count = 0 ;
_reference_frame = image ; _encoded_reference_frame = image ;
encoded_frame = image ; encoded_frame = image ;
differential_frame = false ; differential_frame = false ;
@ -139,23 +195,21 @@ void JPEGVideoEncoder::encodeData(const QImage& image,uint32_t /* size_hint */,u
buffer.open(QIODevice::WriteOnly) ; buffer.open(QIODevice::WriteOnly) ;
encoded_frame.save(&buffer,"JPEG") ; encoded_frame.save(&buffer,"JPEG") ;
RsVOIPDataChunk voip_chunk ;
voip_chunk.data = malloc(HEADER_SIZE + qb.size()); voip_chunk.data = malloc(HEADER_SIZE + qb.size());
// build header // build header
uint32_t flags = differential_frame ? JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME : 0x0 ; uint32_t flags = differential_frame ? JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME : 0x0 ;
((unsigned char *)voip_chunk.data)[0] = flags & 0xff ; ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO & 0xff ;
((unsigned char *)voip_chunk.data)[1] = (flags >> 8) & 0xff ; ((unsigned char *)voip_chunk.data)[1] = (VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO >> 8) & 0xff ;
((unsigned char *)voip_chunk.data)[2] = 0 ; ((unsigned char *)voip_chunk.data)[2] = flags & 0xff ;
((unsigned char *)voip_chunk.data)[3] = 0 ; ((unsigned char *)voip_chunk.data)[3] = (flags >> 8) & 0xff ;
memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ; memcpy(voip_chunk.data+HEADER_SIZE,qb.data(),qb.size()) ;
voip_chunk.size = HEADER_SIZE + qb.size() ; voip_chunk.size = HEADER_SIZE + qb.size() ;
voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
_out_queue.push_back(voip_chunk) ; return true ;
encoded_size = voip_chunk.size ;
} }

168
plugins/VOIP/gui/VideoProcessor.h
View file

@ -6,58 +6,91 @@
class QVideoOutputDevice ; class QVideoOutputDevice ;
class VideoCodec
{
public:
virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) = 0;
virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) = 0;
};
// Now derive various image encoding/decoding algorithms.
//
class JPEGVideo: public VideoCodec
{
public:
JPEGVideo() ;
protected:
virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) ;
virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) ;
static const uint32_t HEADER_SIZE = 0x04 ;
static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ;
private:
QImage _decoded_reference_frame ;
QImage _encoded_reference_frame ;
uint32_t _encoded_ref_frame_max_distance ; // max distance between two reference frames.
uint32_t _encoded_ref_frame_count ;
};
class DifferentialWaveletVideo: public VideoCodec
{
public:
DifferentialWaveletVideo() {}
protected:
virtual bool encodeData(const QImage& Image, uint32_t size_hint, RsVOIPDataChunk& chunk) { return true ; }
virtual bool decodeData(const RsVOIPDataChunk& chunk,QImage& image) { return true ; }
private:
QImage _last_reference_frame ;
};
// This class decodes video from a stream. It keeps a queue of // This class decodes video from a stream. It keeps a queue of
// decoded frame that needs to be retrieved using the getNextImage() method. // decoded frame that needs to be retrieved using the getNextImage() method.
// //
class VideoDecoder class VideoProcessor
{ {
public: public:
VideoDecoder() ; VideoProcessor() ;
virtual ~VideoDecoder() {} virtual ~VideoProcessor() {}
enum CodecId {
VIDEO_PROCESSOR_CODEC_ID_UNKNOWN = 0x0000,
VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO = 0x0001,
VIDEO_PROCESSOR_CODEC_ID_DDWT_VIDEO = 0x0002
};
// =====================================================================================
// =------------------------------------ DECODING -------------------------------------=
// =====================================================================================
// Gets the next image to be displayed. Once returned, the image should // Gets the next image to be displayed. Once returned, the image should
// be cleared from the incoming queue. // be cleared from the incoming queue.
// //
void setDisplayTarget(QVideoOutputDevice *odev) { _output_device = odev ; } void setDisplayTarget(QVideoOutputDevice *odev) { _decoded_output_device = odev ; }
virtual void receiveEncodedData(const RsVOIPDataChunk& chunk) ;
virtual void receiveEncodedData(const unsigned char *data,uint32_t size) ;
// returns the current (measured) frame rate in bytes per second. // returns the current (measured) frame rate in bytes per second.
// //
uint32_t currentFrameRate() const; uint32_t currentDecodingFrameRate() const;
private: private:
QVideoOutputDevice *_output_device ; QVideoOutputDevice *_decoded_output_device ;
std::list<QImage> _decoded_image_queue ;
std::list<QImage> _image_queue ; // =====================================================================================
// =------------------------------------ ENCODING -------------------------------------=
// Incoming data is processed by a video codec and converted into images. // =====================================================================================
//
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) = 0 ;
// // This buffer accumulated incoming encoded data, until a full packet is obtained,
// // since the stream might not send images at once. When incoming images are decoded, the
// // data is removed from the buffer.
// //
// unsigned char *buffer ;
// uint32_t buffer_size ;
};
// This class encodes video using a video codec (possibly homemade, or based on existing codecs)
// and produces a data stream that is sent to the network transfer service (e.g. p3VOIP).
//
class VideoEncoder
{
public: public:
VideoEncoder() ;
virtual ~VideoEncoder() {}
// Takes the next image to be encoded. // Takes the next image to be encoded.
// //
bool addImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ; bool processImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ;
bool encodedPacketReady() const { return !_encoded_out_queue.empty() ; }
bool packetReady() const { return !_out_queue.empty() ; } bool nextEncodedPacket(RsVOIPDataChunk& ) ;
bool nextPacket(RsVOIPDataChunk& ) ;
// Used to tweak the compression ratio so that the video can stream ok. // Used to tweak the compression ratio so that the video can stream ok.
// //
@ -65,63 +98,16 @@ class VideoEncoder
void setInternalFrameSize(QSize) ; void setInternalFrameSize(QSize) ;
protected: protected:
virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) =0; std::list<RsVOIPDataChunk> _encoded_out_queue ;
QSize _encoded_frame_size ;
std::list<RsVOIPDataChunk> _out_queue ;
// =====================================================================================
// =------------------------------------- Codecs --------------------------------------=
// =====================================================================================
QSize _frame_size ; JPEGVideo _jpeg_video_codec ;
DifferentialWaveletVideo _ddwt_video_codec ;
uint16_t _encoding_current_codec ;
}; };
// Now derive various image encoding/decoding algorithms.
//
class JPEGVideoDecoder: public VideoDecoder
{
protected:
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ;
static const uint32_t HEADER_SIZE = 0x04 ;
static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ;
private:
QImage _reference_frame ;
};
class JPEGVideoEncoder: public VideoEncoder
{
public:
JPEGVideoEncoder() ;
protected:
virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ;
static const uint32_t HEADER_SIZE = 0x04 ;
static const uint32_t JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME = 0x0001 ;
private:
QImage _reference_frame ;
uint32_t _ref_frame_max_distance ; // max distance between two reference frames.
uint32_t _ref_frame_count ;
};
class DifferentialWaveletEncoder: public VideoEncoder
{
public:
DifferentialWaveletEncoder() {}
protected:
virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ;
};
class DifferentialWaveletDecoder: public VideoDecoder
{
public:
DifferentialWaveletDecoder() {}
protected:
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ;
private:
QImage _last_reference_frame ;
};