Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2024-12-24 15:09:33 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

added preview option to show decoded stream in Audio config

Browse Source
This commit is contained in:
csoler 2015-08-09 18:09:17 -04:00
parent 2107a1c858
commit c0614e70ac
8 changed files with 195 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
plugins/VOIP/gui/AudioInputConfig.cpp
View File

@ -55,9 +55,9 @@ void AudioInputDialog::showEvent(QShowEvent *) {
class voipGraphSource: public RSGraphSource
{
public:
voipGraphSource() {}
voipGraphSource() : video_input(NULL) {}
void setVideoInput(QVideoInputDevice *vid) { video_input = vid ; }
void setVideoInput(const QVideoInputDevice *vid) { video_input = vid ; }
virtual QString displayName(int) const { return tr("Required bandwidth") ;}
@ -73,21 +73,14 @@ public:
virtual void getValues(std::map<std::string,float>& vals) const
{
RsVOIPDataChunk chunk ;
uint32_t total_size = 0 ;
vals.clear() ;
while(video_input && video_input->getNextEncodedPacket(chunk))
{
total_size += chunk.size ;
chunk.clear() ;
}
vals[std::string("bw")] = (float)total_size ;
if(video_input)
vals[std::string("bw")] = video_input->currentBandwidth() ;
}
private:
QVideoInputDevice *video_input ;
const QVideoInputDevice *video_input ;
};
void voipGraph::setVoipSource(voipGraphSource *gs)
@ -127,12 +120,31 @@ AudioInputConfig::AudioInputConfig(QWidget * parent, Qt::WindowFlags flags)
videoInput->setEchoVideoTarget(ui.videoDisplay) ;
videoInput->setVideoEncoder(new JPEGVideoEncoder()) ;
videoDecoder = new JPEGVideoDecoder;
videoDecoder->setDisplayTarget(NULL) ;
graph_source = new voipGraphSource ;
ui.voipBwGraph->setSource(graph_source);
graph_source->setVideoInput(videoInput) ;
graph_source->setCollectionTimeLimit(1000*300) ;
graph_source->start() ;
QObject::connect(ui.showEncoded_CB,SIGNAL(toggled(bool)),this,SLOT(togglePreview(bool))) ;
}
void AudioInputConfig::togglePreview(bool b)
{
if(b)
{
videoInput->setEchoVideoTarget(NULL) ;
videoDecoder->setDisplayTarget(ui.videoDisplay) ;
}
else
{
videoInput->setEchoVideoTarget(ui.videoDisplay) ;
videoDecoder->setDisplayTarget(NULL) ;
}
}
AudioInputConfig::~AudioInputConfig()
@ -352,6 +364,15 @@ void AudioInputConfig::on_Tick_timeout() {
abSpeech->iValue = iroundf(inputAudioProcessor->dVoiceAcivityLevel * 32767.0f + 0.5f);
abSpeech->update();
// also transmit encoded video
RsVOIPDataChunk chunk ;
while(videoInput->getNextEncodedPacket(chunk))
{
videoDecoder->receiveEncodedData(static_cast<const unsigned char*>(chunk.data),chunk.size) ;
chunk.clear() ;
}
}
void AudioInputConfig::emptyBuffer() {

2
plugins/VOIP/gui/AudioInputConfig.h
View File

@ -69,6 +69,7 @@ class AudioInputConfig : public ConfigPage
//VideoDecoder *videoDecoder ;
//VideoEncoder *videoEncoder ;
QVideoInputDevice *videoInput ;
VideoDecoder *videoDecoder ;
bool loaded;
voipGraphSource *graph_source ;
@ -96,6 +97,7 @@ class AudioInputConfig : public ConfigPage
private slots:
void loadSettings();
void emptyBuffer();
void togglePreview(bool) ;
void on_qsTransmitHold_valueChanged(int v);
void on_qsAmp_valueChanged(int v);

10
plugins/VOIP/gui/AudioInputConfig.ui
View File

@ -391,6 +391,16 @@
</layout>
</widget>
</item>
<item>
<widget class="QCheckBox" name="showEncoded_CB">
<property name="toolTip">
<string>&lt;html&gt;&lt;head/&gt;&lt;body&gt;&lt;p&gt;Display encoded (and then decoded) frame, to check the codec's quality. If not selected, the image above only shows the frame that is grabbed from your camera.&lt;/p&gt;&lt;/body&gt;&lt;/html&gt;</string>
</property>
<property name="text">
<string>preview</string>
</property>
</widget>
</item>
<item>
<spacer>
<property name="orientation">

61
plugins/VOIP/gui/QVideoDevice.cpp
View File

@ -13,6 +13,9 @@ QVideoInputDevice::QVideoInputDevice(QWidget *parent)
_capture_device = NULL ;
_video_encoder = NULL ;
_echo_output_device = NULL ;
_estimated_bw = 0 ;
_total_encoded_size = 0 ;
_last_bw_estimate_TS = time(NULL) ;
}
void QVideoInputDevice::stop()
@ -54,36 +57,52 @@ void QVideoInputDevice::start()
void QVideoInputDevice::grabFrame()
{
IplImage *img=cvQueryFrame(_capture_device);
IplImage *img=cvQueryFrame(_capture_device);
if(img == NULL)
{
std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl;
return ;
}
// get the image data
if(img == NULL)
{
std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl;
return ;
}
// get the image data
if(img->nChannels != 3)
{
std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl;
return ;
}
if(img->nChannels != 3)
{
std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl;
return ;
}
// convert to RGB and copy to new buffer, because cvQueryFrame tells us to not modify the buffer
cv::Mat img_rgb;
cv::cvtColor(cv::Mat(img), img_rgb, CV_BGR2RGB);
static const int _encoded_width = 128 ;
static const int _encoded_height = 128 ;
QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888);
QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888).scaled(QSize(_encoded_width,_encoded_height),Qt::IgnoreAspectRatio,Qt::SmoothTransformation) ;
if(_video_encoder != NULL)
{
uint32_t encoded_size ;
if(_video_encoder != NULL)
{
_video_encoder->addImage(image) ;
emit networkPacketReady() ;
}
if(_echo_output_device != NULL) _echo_output_device->showFrame(image) ;
_video_encoder->addImage(image,0,encoded_size) ;
std::cerr << "Encoded size = " << encoded_size << std::endl;
_total_encoded_size += encoded_size ;
time_t now = time(NULL) ;
if(now > _last_bw_estimate_TS)
{
_estimated_bw = uint32_t(0.75*_estimated_bw + 0.25 * (_total_encoded_size / (float)(now - _last_bw_estimate_TS))) ;
_total_encoded_size = 0 ;
_last_bw_estimate_TS = now ;
std::cerr << "new bw estimate: " << _estimated_bw << std::endl;
}
emit networkPacketReady() ;
}
if(_echo_output_device != NULL)
_echo_output_device->showFrame(image) ;
}
bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk)

10
plugins/VOIP/gui/QVideoDevice.h
View File

@ -42,6 +42,12 @@ class QVideoInputDevice: public QObject
//
bool getNextEncodedPacket(RsVOIPDataChunk&) ;
// gets the estimated current bandwidth required to transmit the encoded data, in B/s
//
uint32_t currentBandwidth() const { return _estimated_bw ; }
// control
void start() ;
void stop() ;
@ -59,5 +65,9 @@ class QVideoInputDevice: public QObject
QVideoOutputDevice *_echo_output_device ;
std::list<RsVOIPDataChunk> _out_queue ;
uint32_t _estimated_bw ;
time_t _last_bw_estimate_TS;
uint32_t _total_encoded_size ;
};

73
plugins/VOIP/gui/VideoProcessor.cpp
View File

@ -12,9 +12,15 @@ VideoDecoder::VideoDecoder()
_output_device = NULL ;
}
bool VideoEncoder::addImage(const QImage& img)
VideoEncoder::VideoEncoder()
:_frame_size(128,128)
{
encodeData(img) ;
}
bool VideoEncoder::addImage(const QImage& img,uint32_t size_hint,uint32_t& encoded_size)
{
std::cerr << "reducing to " << _frame_size.width() << " x " << _frame_size.height() << std::endl;
encodeData(img.scaled(_frame_size,Qt::IgnoreAspectRatio,Qt::SmoothTransformation),size_hint,encoded_size) ;
return true ;
}
@ -32,6 +38,7 @@ bool VideoEncoder::nextPacket(RsVOIPDataChunk& chunk)
void VideoDecoder::receiveEncodedData(const unsigned char *data,uint32_t size)
{
if(_output_device)
_output_device->showFrame(decodeData(data,size)) ;
}
@ -50,24 +57,54 @@ QImage JPEGVideoDecoder::decodeData(const unsigned char *encoded_image_data,uint
void VideoEncoder::setMaximumFrameRate(uint32_t bytes_per_sec)
{
std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl;
std::cerr << "Video Encoder: maximum frame rate is set to " << bytes_per_sec << " Bps" << std::endl;
}
void JPEGVideoEncoder::encodeData(const QImage& image)
void VideoEncoder::setInternalFrameSize(QSize s)
{
QByteArray qb ;
QBuffer buffer(&qb) ;
buffer.open(QIODevice::WriteOnly) ;
image.save(&buffer,"JPEG") ;
RsVOIPDataChunk voip_chunk ;
voip_chunk.data = malloc(qb.size());
memcpy(voip_chunk.data,qb.data(),qb.size()) ;
voip_chunk.size = qb.size() ;
voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
_out_queue.push_back(voip_chunk) ;
_frame_size = s ;
}
JPEGVideoEncoder::JPEGVideoEncoder()
: _ref_frame_max_distance(10),_ref_frame_count(10)
{
}
void JPEGVideoEncoder::encodeData(const QImage& image,uint32_t /* size_hint */,uint32_t& encoded_size)
{
// check if we make a diff image, or if we use the full frame.
QImage encoded_frame ;
if(_ref_frame_count++ < _ref_frame_max_distance && image.size() == _reference_frame.size())
{
// compute difference with reference frame.
encoded_frame = image ;
for(uint32_t i=0;i<image.byteCount();++i)
encoded_frame.bits()[i] = image.bits()[i] - _reference_frame.bits()[i] + 128 ;
}
else
{
_ref_frame_count = 0 ;
_reference_frame = image ;
encoded_frame = image ;
}
QByteArray qb ;
QBuffer buffer(&qb) ;
buffer.open(QIODevice::WriteOnly) ;
encoded_frame.save(&buffer,"JPEG") ;
RsVOIPDataChunk voip_chunk ;
voip_chunk.data = malloc(qb.size());
memcpy(voip_chunk.data,qb.data(),qb.size()) ;
voip_chunk.size = qb.size() ;
voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ;
_out_queue.push_back(voip_chunk) ;
encoded_size = voip_chunk.size ;
}

53
plugins/VOIP/gui/VideoProcessor.h
View File

@ -49,12 +49,12 @@ class VideoDecoder
class VideoEncoder
{
public:
VideoEncoder() {}
VideoEncoder() ;
virtual ~VideoEncoder() {}
// Takes the next image to be encoded.
//
bool addImage(const QImage& Image) ;
bool addImage(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ;
bool packetReady() const { return !_out_queue.empty() ; }
bool nextPacket(RsVOIPDataChunk& ) ;
@ -62,12 +62,14 @@ class VideoEncoder
// Used to tweak the compression ratio so that the video can stream ok.
//
void setMaximumFrameRate(uint32_t bytes_per_second) ;
void setInternalFrameSize(QSize) ;
protected:
//virtual bool sendEncodedData(unsigned char *mem,uint32_t size) = 0 ;
virtual void encodeData(const QImage& image) = 0 ;
virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) =0;
std::list<RsVOIPDataChunk> _out_queue ;
QSize _frame_size ;
};
// Now derive various image encoding/decoding algorithms.
@ -75,16 +77,45 @@ class VideoEncoder
class JPEGVideoDecoder: public VideoDecoder
{
protected:
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ;
protected:
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ;
private:
QImage _last_reference_frame ;
};
class JPEGVideoEncoder: public VideoEncoder
{
public:
JPEGVideoEncoder() {}
public:
JPEGVideoEncoder() ;
protected:
virtual void encodeData(const QImage& Image) ;
protected:
virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ;
private:
QImage _reference_frame ;
uint32_t _ref_frame_max_distance ; // max distance between two reference frames.
uint32_t _ref_frame_count ;
};
class DifferentialWaveletEncoder: public VideoEncoder
{
public:
DifferentialWaveletEncoder() {}
protected:
virtual void encodeData(const QImage& Image, uint32_t size_hint, uint32_t &encoded_size) ;
};
class DifferentialWaveletDecoder: public VideoDecoder
{
public:
DifferentialWaveletDecoder() {}
protected:
virtual QImage decodeData(const unsigned char *encoded_image,uint32_t encoded_image_size) ;
private:
QImage _last_reference_frame ;
};

4
plugins/VOIP/interface/rsVOIP.h
View File

@ -51,7 +51,9 @@ class RsVOIPPongResult
struct RsVOIPDataChunk
{
typedef enum { RS_VOIP_DATA_TYPE_AUDIO, RS_VOIP_DATA_TYPE_VIDEO } RsVOIPDataType ;
typedef enum { RS_VOIP_DATA_TYPE_UNKNOWN = 0x00,
RS_VOIP_DATA_TYPE_AUDIO = 0x01,
RS_VOIP_DATA_TYPE_VIDEO = 0x02 } RsVOIPDataType ;
void *data ; // create/delete using malloc/free.
uint32_t size ;