diff --git a/plugins/VOIP/VOIP.pro b/plugins/VOIP/VOIP.pro index dde97ea93..746530290 100644 --- a/plugins/VOIP/VOIP.pro +++ b/plugins/VOIP/VOIP.pro @@ -30,11 +30,52 @@ linux-* { win32 { - LIBS_DIR = $$PWD/../../../libs - LIBS += -L"$$LIBS_DIR/lib/opencv" + DEPENDPATH += . $$INC_DIR + INCLUDEPATH += . $$INC_DIR - OPENCV_VERSION = 249 - LIBS += -lopencv_core$$OPENCV_VERSION -lopencv_highgui$$OPENCV_VERSION -lopencv_imgproc$$OPENCV_VERSION -llibjpeg -llibtiff -llibpng -llibjasper -lIlmImf -lole32 -loleaut32 -luuid -lavicap32 -lavifil32 -lvfw32 -lz + OPENCV_VERSION = "249" + USE_PRECOMPILED_LIBS = + for(lib, LIB_DIR) { +#message(Scanning $$lib) + exists( $$lib/opencv/libopencv_core249.a) { + isEmpty(USE_PRECOMPILED_LIBS) { + message(Get pre-compiled opencv 249 libraries here:) + message($$lib) + LIBS += -L"$$lib/opencv" + LIBS += -lopencv_core$$OPENCV_VERSION -lopencv_highgui$$OPENCV_VERSION -lopencv_imgproc$$OPENCV_VERSION + USE_PRECOMPILED_LIBS = 1 + } + } + exists( $$lib/opencv/libopencv_core.a) { + isEmpty(USE_PRECOMPILED_LIBS) { + message(Get pre-compiled opencv libraries here:) + message($$lib) + LIBS += -L"$$lib/opencv" + LIBS += -lopencv_core -lopencv_highgui -lopencv_imgproc + USE_PRECOMPILED_LIBS = 1 + } + } + exists( $$lib/libopencv_core.dll.a) { + isEmpty(USE_PRECOMPILED_LIBS) { + message(Get pre-compiled opencv libraries here:) + message($$lib) + LIBS += -L"$$lib/opencv" + LIBS += -lopencv_core -lopencv_highgui -lopencv_imgproc + USE_PRECOMPILED_LIBS = 1 + } + } + exists( $$lib/libopencv_videoio.dll.a) { + message(videoio found in opencv libraries.) + message($$lib) + LIBS += -lopencv_videoio + } + } + isEmpty(USE_PRECOMPILED_LIBS) { + message(Use system opencv libraries.) + LIBS += -lopencv_core -lopencv_highgui -lopencv_imgproc + } + LIBS += -lz -lole32 -loleaut32 -luuid -lvfw32 -llibjpeg -llibtiff -llibpng -llibjasper -lIlmImf + LIBS += -lavifil32 -lavicap32 -lavcodec -lavutil -lswresample } # ffmpeg (and libavutil: https://github.com/ffms/ffms2/issues/11) diff --git a/plugins/VOIP/gui/AudioInputConfig.cpp b/plugins/VOIP/gui/AudioInputConfig.cpp index d1503040b..b2add04e6 100644 --- a/plugins/VOIP/gui/AudioInputConfig.cpp +++ b/plugins/VOIP/gui/AudioInputConfig.cpp @@ -140,18 +140,18 @@ void AudioInputConfig::updateAvailableBW(double r) std::cerr << "Setting max bandwidth to " << r << " KB/s" << std::endl; videoProcessor->setMaximumBandwidth((uint32_t)(r*1024)) ; } - + void AudioInputConfig::togglePreview(bool b) { if(b) { videoInput->setEchoVideoTarget(NULL) ; - videoProcessor->setDisplayTarget(ui.videoDisplay) ; + videoProcessor->setDisplayTarget(ui.videoDisplay) ; } else { + videoProcessor->setDisplayTarget(NULL) ; videoInput->setEchoVideoTarget(ui.videoDisplay) ; - videoProcessor->setDisplayTarget(NULL) ; } } diff --git a/plugins/VOIP/gui/QVideoDevice.cpp b/plugins/VOIP/gui/QVideoDevice.cpp index b81b7b79c..5a355e085 100644 --- a/plugins/VOIP/gui/QVideoDevice.cpp +++ b/plugins/VOIP/gui/QVideoDevice.cpp @@ -31,21 +31,23 @@ void QVideoInputDevice::stop() } if(_capture_device != NULL) { - cvReleaseCapture(&_capture_device) ; + // the camera will be deinitialized automatically in VideoCapture destructor + _capture_device->release(); + delete _capture_device ; _capture_device = NULL ; } } void QVideoInputDevice::start() { - // make sure everything is re-initialised + // make sure everything is re-initialised // stop() ; - // Initialise la capture + // Initialise la capture static const int cam_id = 0 ; - _capture_device = cvCaptureFromCAM(cam_id); + _capture_device = new cv::VideoCapture(cam_id); - if(_capture_device == NULL) + if(!_capture_device->isOpened()) { std::cerr << "Cannot initialise camera. Something's wrong." << std::endl; return ; @@ -61,52 +63,51 @@ void QVideoInputDevice::grabFrame() { if(!_timer) return ; - - IplImage *img=cvQueryFrame(_capture_device); - if(img == NULL) + cv::Mat frame; + if(!_capture_device->read(frame)) { - std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; - return ; + std::cerr << "(EE) Cannot capture image from camera. Something's wrong." << std::endl; + return ; } + // get the image data - if(img->nChannels != 3) + if(frame.channels() != 3) { - std::cerr << "(EE) expected 3 channels. Got " << img->nChannels << std::endl; - return ; + std::cerr << "(EE) expected 3 channels. Got " << frame.channels() << std::endl; + return ; } // convert to RGB and copy to new buffer, because cvQueryFrame tells us to not modify the buffer cv::Mat img_rgb; - cv::cvtColor(cv::Mat(img), img_rgb, CV_BGR2RGB); - + cv::cvtColor(frame, img_rgb, CV_BGR2RGB); QImage image = QImage(img_rgb.data,img_rgb.cols,img_rgb.rows,QImage::Format_RGB888); - if(_video_processor != NULL) + if(_video_processor != NULL) { - _video_processor->processImage(image) ; + _video_processor->processImage(image) ; - emit networkPacketReady() ; + emit networkPacketReady() ; } - if(_echo_output_device != NULL) - _echo_output_device->showFrame(image) ; + if(_echo_output_device != NULL) + _echo_output_device->showFrame(image) ; } bool QVideoInputDevice::getNextEncodedPacket(RsVOIPDataChunk& chunk) { if(!_timer) return false ; - + if(_video_processor) - return _video_processor->nextEncodedPacket(chunk) ; - else + return _video_processor->nextEncodedPacket(chunk) ; + else return false ; } uint32_t QVideoInputDevice::currentBandwidth() const { - return _video_processor->currentBandwidthOut() ; + return _video_processor->currentBandwidthOut() ; } QVideoInputDevice::~QVideoInputDevice() @@ -117,7 +118,7 @@ QVideoInputDevice::~QVideoInputDevice() QVideoOutputDevice::QVideoOutputDevice(QWidget *parent) - : QLabel(parent) + : QLabel(parent) { showFrameOff() ; } @@ -130,6 +131,6 @@ void QVideoOutputDevice::showFrameOff() void QVideoOutputDevice::showFrame(const QImage& img) { std::cerr << "img.size = " << img.width() << " x " << img.height() << std::endl; - setPixmap(QPixmap::fromImage(img).scaled( QSize(height()*640/480,height()),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; + setPixmap(QPixmap::fromImage(img).scaled( QSize(height()*4/3,height()),Qt::IgnoreAspectRatio,Qt::SmoothTransformation)) ; } diff --git a/plugins/VOIP/gui/QVideoDevice.h b/plugins/VOIP/gui/QVideoDevice.h index cc92302a2..f1f4659ba 100644 --- a/plugins/VOIP/gui/QVideoDevice.h +++ b/plugins/VOIP/gui/QVideoDevice.h @@ -2,10 +2,12 @@ #include #include "interface/rsVOIP.h" + +#include "opencv2/opencv.hpp" + #include "gui/VideoProcessor.h" class VideoEncoder ; -class CvCapture ; // Responsible from displaying the video. The source of the video is // a VideoDecoder object, which uses a codec. @@ -61,7 +63,7 @@ protected slots: private: VideoProcessor *_video_processor ; QTimer *_timer ; - CvCapture *_capture_device ; + cv::VideoCapture *_capture_device ; QVideoOutputDevice *_echo_output_device ; diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index 7fbf9c2d6..412e16da5 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -12,10 +12,6 @@ #include #include -#if defined(__MINGW32__) -#define memalign _aligned_malloc -#endif //MINGW - extern "C" { #include @@ -28,21 +24,94 @@ extern "C" { } //#define DEBUG_MPEG_VIDEO 1 +#ifndef AV_INPUT_BUFFER_PADDING_SIZE +#ifndef FF_INPUT_BUFFER_PADDING_SIZE +#define AV_INPUT_BUFFER_PADDING_SIZE 32 +#else +#define AV_INPUT_BUFFER_PADDING_SIZE FF_INPUT_BUFFER_PADDING_SIZE +#endif +#endif + +#if (LIBAVUTIL_VERSION_MAJOR < 52) || ((LIBAVUTIL_VERSION_MAJOR == 52) && (LIBAVUTIL_VERSION_MINOR < 63)) +//since https://github.com/FFmpeg/FFmpeg/commit/3532dd52c51f3d4b95f31d1b195e64a04a8aea5d +static inline AVRational av_make_q(int num, int den) +{ + AVRational r = { num, den }; + return r; +} +#endif + +#if (LIBAVUTIL_VERSION_MAJOR < 55) || ((LIBAVUTIL_VERSION_MAJOR == 55) && (LIBAVUTIL_VERSION_MINOR < 52)) +//since https://github.com/FFmpeg/FFmpeg/commit/fd056029f45a9f6d213d9fce8165632042511d4f +void avcodec_free_context(AVCodecContext **pavctx) +{ + AVCodecContext *avctx = *pavctx; + + if (!avctx) + return; + + avcodec_close(avctx); + + av_freep(&avctx->extradata); + av_freep(&avctx->subtitle_header); + + av_freep(pavctx); +} +#endif + + +#if (LIBAVUTIL_VERSION_MAJOR < 57) || ((LIBAVUTIL_VERSION_MAJOR == 57) && (LIBAVUTIL_VERSION_MINOR < 52)) +//Since https://github.com/FFmpeg/FFmpeg/commit/7ecc2d403ce5c7b6ea3b1f368dccefd105209c7e +static void get_frame_defaults(AVFrame *frame) +{ + if (frame->extended_data != frame->data) + av_freep(&frame->extended_data); + return; + + frame->extended_data = NULL; + get_frame_defaults(frame); + + return; +} + +AVFrame *av_frame_alloc(void) +{ + AVFrame *frame = (AVFrame *)av_mallocz(sizeof(*frame)); + + if (!frame) + return NULL; + + get_frame_defaults(frame); + + return frame; +} + + +void av_frame_free(AVFrame **frame) +{ + if (!frame || !*frame) + return; + + av_freep(frame); +} +#endif + + VideoProcessor::VideoProcessor() :_encoded_frame_size(640,480) , vpMtx("VideoProcessor") { _decoded_output_device = NULL ; - + //_encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO; _encoding_current_codec = VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO; - + _estimated_bandwidth_in = 0 ; _estimated_bandwidth_out = 0 ; _target_bandwidth_out = 30*1024 ; // 30 KB/s - + _total_encoded_size_in = 0 ; _total_encoded_size_out = 0 ; - + _last_bw_estimate_in_TS = time(NULL) ; _last_bw_estimate_out_TS = time(NULL) ; } @@ -50,9 +119,9 @@ VideoProcessor::VideoProcessor() VideoProcessor::~VideoProcessor() { // clear encoding queue - + RS_STACK_MUTEX(vpMtx) ; - + while(!_encoded_out_queue.empty()) { _encoded_out_queue.back().clear() ; @@ -92,7 +161,7 @@ bool VideoProcessor::processImage(const QImage& img) if(now > _last_bw_estimate_out_TS) { RS_STACK_MUTEX(vpMtx) ; - + _estimated_bandwidth_out = uint32_t(0.75*_estimated_bandwidth_out + 0.25 * (_total_encoded_size_out / (float)(now - _last_bw_estimate_out_TS))) ; _total_encoded_size_out = 0 ; @@ -114,7 +183,7 @@ bool VideoProcessor::processImage(const QImage& img) bool VideoProcessor::nextEncodedPacket(RsVOIPDataChunk& chunk) { - RS_STACK_MUTEX(vpMtx) ; + RS_STACK_MUTEX(vpMtx) ; if(_encoded_out_queue.empty()) return false ; @@ -134,7 +203,7 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) static const int HEADER_SIZE = 4 ; // read frame type. Use first 4 bytes to give info about content. - // + // // Byte Meaning Values // 00 Codec CODEC_ID_JPEG_VIDEO Basic Jpeg codec // CODEC_ID_DDWT_VIDEO Differential wavelet compression @@ -164,14 +233,13 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) default: codec = NULL ; } - QImage img ; if(codec == NULL) { std::cerr << "Unknown decoding codec: " << codid << std::endl; return ; } - + { RS_STACK_MUTEX(vpMtx) ; _total_encoded_size_in += chunk.size ; @@ -188,16 +256,18 @@ void VideoProcessor::receiveEncodedData(const RsVOIPDataChunk& chunk) #ifdef DEBUG_VIDEO_OUTPUT_DEVICE std::cerr << "new bw estimate (in): " << _estimated_bandwidth_in << std::endl; #endif - } + } } - if(!codec->decodeData(chunk,img)) + + QImage img ; + if(!codec->decodeData(chunk,img)) { std::cerr << "No image decoded. Probably in the middle of something..." << std::endl; return ; } if(_decoded_output_device) - _decoded_output_device->showFrame(img) ; + _decoded_output_device->showFrame(img) ; } void VideoProcessor::setMaximumBandwidth(uint32_t bytes_per_sec) @@ -211,7 +281,7 @@ void VideoProcessor::setMaximumBandwidth(uint32_t bytes_per_sec) ////////////////////////////////////////////////////////////////////////////////////////////////////////////// JPEGVideo::JPEGVideo() - : _encoded_ref_frame_max_distance(10),_encoded_ref_frame_count(10) + : _encoded_ref_frame_max_distance(10),_encoded_ref_frame_count(10) { } @@ -221,9 +291,9 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) uint16_t codec = ((unsigned char *)chunk.data)[0] + (((unsigned char *)chunk.data)[1] << 8) ; uint16_t flags = ((unsigned char *)chunk.data)[2] + (((unsigned char *)chunk.data)[3] << 8) ; - + assert(codec == VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_JPEG_VIDEO) ; - + // un-compress image data QByteArray qb((char*)&((uint8_t*)chunk.data)[HEADER_SIZE],(int)chunk.size - HEADER_SIZE) ; @@ -233,7 +303,7 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) std::cerr << "image.loadFromData(): returned an error.: " << std::endl; return false ; } - + if(flags & JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME) { if(_decoded_reference_frame.size() != image.size()) @@ -255,7 +325,7 @@ bool JPEGVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) } else _decoded_reference_frame = image ; - + return true ; } @@ -266,8 +336,10 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa QImage encoded_frame ; bool differential_frame ; - if(_encoded_ref_frame_count++ < _encoded_ref_frame_max_distance && image.size() == _encoded_reference_frame.size()) - { + if (_encoded_ref_frame_count++ < _encoded_ref_frame_max_distance + && image.size() == _encoded_reference_frame.size() + && image.byteCount() == _encoded_reference_frame.byteCount()) + { // compute difference with reference frame. encoded_frame = image ; @@ -285,7 +357,7 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa else { _encoded_ref_frame_count = 0 ; - _encoded_reference_frame = image ; + _encoded_reference_frame = image.copy() ; encoded_frame = image ; differential_frame = false ; @@ -317,13 +389,14 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa FFmpegVideo::FFmpegVideo() { - // Encoding - + avcodec_register_all(); + // Encoding + encoding_codec = NULL ; encoding_frame_buffer = NULL ; encoding_context = NULL ; - - //AVCodecID codec_id = AV_CODEC_ID_H264 ; + + //AVCodecID codec_id = AV_CODEC_ID_H264 ; //AVCodecID codec_id = AV_CODEC_ID_MPEG2VIDEO; #if LIBAVCODEC_VERSION_MAJOR < 54 CodecID codec_id = CODEC_ID_MPEG4; @@ -331,19 +404,21 @@ FFmpegVideo::FFmpegVideo() AVCodecID codec_id = AV_CODEC_ID_MPEG4; #endif - /* find the mpeg1 video encoder */ + /* find the video encoder */ encoding_codec = avcodec_find_encoder(codec_id); - - if (!encoding_codec) throw("AV codec not found for codec id ") ; + + if (!encoding_codec) std::cerr << "AV codec not found for codec id " << std::endl; + if (!encoding_codec) throw std::runtime_error("AV codec not found for codec id ") ; encoding_context = avcodec_alloc_context3(encoding_codec); - - if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); + + if (!encoding_context) std::cerr << "AV: Could not allocate video codec encoding context" << std::endl; + if (!encoding_context) throw std::runtime_error("AV: Could not allocate video codec encoding context"); /* put sample parameters */ encoding_context->bit_rate = 10*1024 ; // default bitrate is 30KB/s encoding_context->bit_rate_tolerance = encoding_context->bit_rate ; - + #ifdef USE_VARIABLE_BITRATE encoding_context->rc_min_rate = 0; encoding_context->rc_max_rate = 10*1024;//encoding_context->bit_rate; @@ -356,8 +431,9 @@ FFmpegVideo::FFmpegVideo() encoding_context->rc_max_rate = 0; encoding_context->rc_buffer_size = 0; #endif - encoding_context->flags |= CODEC_FLAG_PSNR; - encoding_context->flags |= CODEC_FLAG_TRUNCATED; + if (encoding_codec->capabilities & CODEC_CAP_TRUNCATED) + encoding_context->flags |= CODEC_FLAG_TRUNCATED; + encoding_context->flags |= CODEC_FLAG_PSNR;//Peak signal-to-noise ratio encoding_context->flags |= CODEC_CAP_PARAM_CHANGE; encoding_context->i_quant_factor = 0.769f; encoding_context->b_quant_factor = 1.4f; @@ -366,19 +442,19 @@ FFmpegVideo::FFmpegVideo() encoding_context->qmin = 1; encoding_context->qmax = 51; encoding_context->max_qdiff = 4; - + //encoding_context->me_method = ME_HEX; - //encoding_context->max_b_frames = 4; + //encoding_context->max_b_frames = 4; //encoding_context->flags |= CODEC_FLAG_LOW_DELAY; // MPEG2 only //encoding_context->partitions = X264_PART_I4X4 | X264_PART_I8X8 | X264_PART_P8X8 | X264_PART_P4X4 | X264_PART_B8X8; //encoding_context->crf = 0.0f; //encoding_context->cqp = 26; - + /* resolution must be a multiple of two */ encoding_context->width = 640;//176; encoding_context->height = 480;//144; /* frames per second */ - encoding_context->time_base = (AVRational){1,25}; + encoding_context->time_base = av_make_q(1, 25); /* emit one intra frame every ten frames * check frame pict_type before passing frame * to encoder, if frame->pict_type is AV_PICTURE_TYPE_I @@ -398,41 +474,51 @@ FFmpegVideo::FFmpegVideo() } /* open it */ - if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0) + if (avcodec_open2(encoding_context, encoding_codec, NULL) < 0) + { + std::cerr << "AV: Could not open codec context. Something's wrong." << std::endl; throw std::runtime_error( "AV: Could not open codec context. Something's wrong."); + } +#if (LIBAVCODEC_VERSION_MAJOR < 57) | (LIBAVCODEC_VERSION_MAJOR == 57 && LIBAVCODEC_VERSION_MINOR <3 ) encoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; - +#else + encoding_frame_buffer = av_frame_alloc() ; +#endif + + if(!encoding_frame_buffer) std::cerr << "AV: could not allocate frame buffer." << std::endl; if(!encoding_frame_buffer) throw std::runtime_error("AV: could not allocate frame buffer.") ; - + encoding_frame_buffer->format = encoding_context->pix_fmt; encoding_frame_buffer->width = encoding_context->width; encoding_frame_buffer->height = encoding_context->height; /* the image can be allocated by any means and av_image_alloc() is * just the most convenient way if av_malloc() is to be used */ - + int ret = av_image_alloc(encoding_frame_buffer->data, encoding_frame_buffer->linesize, encoding_context->width, encoding_context->height, encoding_context->pix_fmt, 32); - - if (ret < 0) - throw std::runtime_error("AV: Could not allocate raw picture buffer"); - + + if (ret < 0) std::cerr << "AV: Could not allocate raw picture buffer" << std::endl; + if (ret < 0) + throw std::runtime_error("AV: Could not allocate raw picture buffer"); + encoding_frame_count = 0 ; - + // Decoding - decoding_codec = avcodec_find_decoder(codec_id); - - if (!decoding_codec) + + if (!decoding_codec) std::cerr << "AV codec not found for codec id " << std::endl; + if (!decoding_codec) throw("AV codec not found for codec id ") ; - + decoding_context = avcodec_alloc_context3(decoding_codec); - - if(!decoding_context) + + if(!decoding_context) std::cerr << "AV: Could not allocate video codec decoding context" << std::endl; + if(!decoding_context) throw std::runtime_error("AV: Could not allocate video codec decoding context"); - + decoding_context->width = encoding_context->width; decoding_context->height = encoding_context->height; #if LIBAVCODEC_VERSION_MAJOR < 54 @@ -440,57 +526,60 @@ FFmpegVideo::FFmpegVideo() #else decoding_context->pix_fmt = AV_PIX_FMT_YUV420P; #endif - + if(decoding_codec->capabilities & CODEC_CAP_TRUNCATED) - decoding_context->flags |= CODEC_FLAG_TRUNCATED; // we do not send complete frames - - if(avcodec_open2(decoding_context,decoding_codec,NULL) < 0) + decoding_context->flags |= CODEC_FLAG_TRUNCATED; // we do not send complete frames + //we can receive truncated frames + decoding_context->flags2 |= CODEC_FLAG2_CHUNKS; + + AVDictionary* dictionary = NULL; + if(avcodec_open2(decoding_context, decoding_codec, &dictionary) < 0) + { + std::cerr << "AV codec open action failed! " << std::endl; throw("AV codec open action failed! ") ; - - decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; - + } + + //decoding_frame_buffer = avcodec_alloc_frame() ;//(AVFrame*)malloc(sizeof(AVFrame)) ; + decoding_frame_buffer = av_frame_alloc() ; + av_init_packet(&decoding_buffer); decoding_buffer.data = NULL ; decoding_buffer.size = 0 ; //ret = av_image_alloc(decoding_frame_buffer->data, decoding_frame_buffer->linesize, decoding_context->width, decoding_context->height, decoding_context->pix_fmt, 32); - - //if (ret < 0) - //throw std::runtime_error("AV: Could not allocate raw picture buffer"); - + + //if (ret < 0) + //throw std::runtime_error("AV: Could not allocate raw picture buffer"); + // debug -#ifdef DEBUG_MPEG_VIDEO +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Dumping captured data to file tmpvideo.mpg" << std::endl; encoding_debug_file = fopen("tmpvideo.mpg","w") ; #endif } - + FFmpegVideo::~FFmpegVideo() { - avcodec_close(encoding_context); - avcodec_close(decoding_context); - av_free(encoding_context); - av_free(decoding_context); - av_freep(&encoding_frame_buffer->data[0]); - av_freep(&decoding_frame_buffer->data[0]); - free(encoding_frame_buffer); - free(decoding_frame_buffer); + avcodec_free_context(&encoding_context); + avcodec_free_context(&decoding_context); + av_frame_free(&encoding_frame_buffer); + av_frame_free(&decoding_frame_buffer); } #define MAX_FFMPEG_ENCODING_BITRATE 81920 bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitrate, RsVOIPDataChunk& voip_chunk) { -#ifdef DEBUG_MPEG_VIDEO - std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; +#ifdef DEBUG_MPEG_VIDEO + std::cerr << "Encoding frame of size " << image.width() << "x" << image.height() << ", resized to " << encoding_frame_buffer->width << "x" << encoding_frame_buffer->height << " : "; #endif QImage input ; - - if(target_encoding_bitrate > MAX_FFMPEG_ENCODING_BITRATE) - { - std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl; - target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ; - } + + if(target_encoding_bitrate > MAX_FFMPEG_ENCODING_BITRATE) + { + std::cerr << "Max encodign bitrate eexceeded. Capping to " << MAX_FFMPEG_ENCODING_BITRATE << std::endl; + target_encoding_bitrate = MAX_FFMPEG_ENCODING_BITRATE ; + } //encoding_context->bit_rate = target_encoding_bitrate; encoding_context->rc_max_rate = target_encoding_bitrate; //encoding_context->bit_rate_tolerance = target_encoding_bitrate; @@ -502,8 +591,8 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra /* prepare a dummy image */ /* Y */ - for (int y = 0; y < encoding_context->height/2; y++) - for (int x = 0; x < encoding_context->width/2; x++) + for (int y = 0; y < encoding_context->height/2; y++) + for (int x = 0; x < encoding_context->width/2; x++) { QRgb pix00 = input.pixel(QPoint(2*x+0,2*y+0)) ; QRgb pix01 = input.pixel(QPoint(2*x+0,2*y+1)) ; @@ -519,11 +608,11 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra int Y01 = (0.257 * R01) + (0.504 * G01) + (0.098 * B01) + 16 ; int Y10 = (0.257 * R10) + (0.504 * G10) + (0.098 * B10) + 16 ; int Y11 = (0.257 * R11) + (0.504 * G11) + (0.098 * B11) + 16 ; - - float R = 0.25*(R00+R01+R10+R11) ; - float G = 0.25*(G00+G01+G10+G11) ; - float B = 0.25*(B00+B01+B10+B11) ; - + + float R = 0.25*(R00+R01+R10+R11) ; + float G = 0.25*(G00+G01+G10+G11) ; + float B = 0.25*(B00+B01+B10+B11) ; + int U = (0.439 * R) - (0.368 * G) - (0.071 * B) + 128 ; int V = -(0.148 * R) - (0.291 * G) + (0.439 * B) + 128 ; @@ -531,7 +620,7 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra encoding_frame_buffer->data[0][(2*y+0) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y01)); // Y encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+0] = std::min(255,std::max(0,Y10)); // Y encoding_frame_buffer->data[0][(2*y+1) * encoding_frame_buffer->linesize[0] + 2*x+1] = std::min(255,std::max(0,Y11)); // Y - + encoding_frame_buffer->data[1][y * encoding_frame_buffer->linesize[1] + x] = std::min(255,std::max(0,U));// Cr encoding_frame_buffer->data[2][y * encoding_frame_buffer->linesize[2] + x] = std::min(255,std::max(0,V));// Cb } @@ -543,8 +632,6 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra int got_output = 0; - AVFrame *frame = encoding_frame_buffer ; - AVPacket pkt ; av_init_packet(&pkt); #if LIBAVCODEC_VERSION_MAJOR < 54 @@ -553,7 +640,7 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra // do // { - int ret = avcodec_encode_video(encoding_context, pkt.data, pkt.size, frame) ; + int ret = avcodec_encode_video(encoding_context, pkt.data, pkt.size, encoding_frame_buffer) ; if (ret > 0) { got_output = ret; } @@ -563,10 +650,10 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra // do // { - int ret = avcodec_encode_video2(encoding_context, &pkt, frame, &got_output) ; + int ret = avcodec_encode_video2(encoding_context, &pkt, encoding_frame_buffer, &got_output) ; #endif - if (ret < 0) + if (ret < 0) { std::cerr << "Error encoding frame!" << std::endl; return false ; @@ -590,14 +677,14 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra voip_chunk.size = pkt.size + HEADER_SIZE; voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; -#ifdef DEBUG_MPEG_VIDEO +#ifdef DEBUG_MPEG_VIDEO std::cerr << "Output : " << pkt.size << " bytes." << std::endl; fwrite(pkt.data,1,pkt.size,encoding_debug_file) ; fflush(encoding_debug_file) ; #endif av_free_packet(&pkt); - - return true ; + + return true ; } else { @@ -606,7 +693,7 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra voip_chunk.type = RsVOIPDataChunk::RS_VOIP_DATA_TYPE_VIDEO ; std::cerr << "No output produced." << std::endl; - return false ; + return false ; } } @@ -615,93 +702,68 @@ bool FFmpegVideo::decodeData(const RsVOIPDataChunk& chunk,QImage& image) { #ifdef DEBUG_MPEG_VIDEO std::cerr << "Decoding data of size " << chunk.size << std::endl; -#endif - unsigned char *buff ; - - if(decoding_buffer.data != NULL) - { -#ifdef DEBUG_MPEG_VIDEO - std::cerr << "Completing buffer with size " << chunk.size - HEADER_SIZE + decoding_buffer.size << ": copying existing " - << decoding_buffer.size << " bytes. Adding new " << chunk.size - HEADER_SIZE<< " bytes " << std::endl; + std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl; #endif - uint32_t s = chunk.size - HEADER_SIZE + decoding_buffer.size ; - unsigned char *tmp = (unsigned char*)memalign(16,s + FF_INPUT_BUFFER_PADDING_SIZE) ; - memset(tmp,0,s+FF_INPUT_BUFFER_PADDING_SIZE) ; - - memcpy(tmp,decoding_buffer.data,decoding_buffer.size) ; - - free(decoding_buffer.data) ; - - buff = &tmp[decoding_buffer.size] ; - decoding_buffer.size = s ; - decoding_buffer.data = tmp ; + uint32_t s = chunk.size - HEADER_SIZE ; +#if defined(__MINGW32__) + unsigned char *tmp = (unsigned char*)_aligned_malloc(s + AV_INPUT_BUFFER_PADDING_SIZE, 16) ; +#else + unsigned char *tmp = (unsigned char*)memalign(16, s + AV_INPUT_BUFFER_PADDING_SIZE) ; +#endif //MINGW + if (tmp == NULL) { + std::cerr << "FFmpegVideo::decodeData() Unable to allocate new buffer of size " << s << std::endl; + return false; } - else - { -#ifdef DEBUG_MPEG_VIDEO - std::cerr << "Allocating new buffer of size " << chunk.size - HEADER_SIZE << std::endl; -#endif + /* copy chunk data without header to new buffer */ + memcpy(tmp, &((unsigned char*)chunk.data)[HEADER_SIZE], s); - decoding_buffer.data = (unsigned char *)memalign(16,chunk.size - HEADER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE) ; - decoding_buffer.size = chunk.size - HEADER_SIZE ; - memset(decoding_buffer.data,0,decoding_buffer.size + FF_INPUT_BUFFER_PADDING_SIZE) ; + /* set end of buffer to 0 (this ensures that no overreading happens for damaged mpeg streams) */ + memset(&tmp[s], 0, AV_INPUT_BUFFER_PADDING_SIZE) ; - buff = decoding_buffer.data ; - } + decoding_buffer.size = s ; + decoding_buffer.data = tmp; + int got_frame = 1 ; - memcpy(buff,&((unsigned char*)chunk.data)[HEADER_SIZE],chunk.size - HEADER_SIZE) ; + while (decoding_buffer.size > 0 || (!decoding_buffer.data && got_frame)) { + int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; - int got_frame = 0 ; - int len = avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; + if (len < 0) + { + std::cerr << "Error decoding frame! Return=" << len << std::endl; + return false ; + } - if(len < 0) - { - std::cerr << "Error decodign frame!" << std::endl; - return false ; - } -#ifdef DEBUG_MPEG_VIDEO - std::cerr << "Used " << len << " bytes out of " << decoding_buffer.size << ". got_frame = " << got_frame << std::endl; -#endif + decoding_buffer.data += len; + decoding_buffer.size -= len; - if(got_frame) - { - image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; + if(got_frame) + { + image = QImage(QSize(decoding_frame_buffer->width,decoding_frame_buffer->height),QImage::Format_ARGB32) ; #ifdef DEBUG_MPEG_VIDEO - std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; + std::cerr << "Decoded frame. Size=" << image.width() << "x" << image.height() << std::endl; #endif - for (int y = 0; y < decoding_frame_buffer->height; y++) - for (int x = 0; x < decoding_frame_buffer->width; x++) - { - int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; - int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ; - int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ; + for (int y = 0; y < decoding_frame_buffer->height; y++) + for (int x = 0; x < decoding_frame_buffer->width; x++) + { + int Y = decoding_frame_buffer->data[0][y * decoding_frame_buffer->linesize[0] + x] ; + int U = decoding_frame_buffer->data[1][(y/2) * decoding_frame_buffer->linesize[1] + x/2] ; + int V = decoding_frame_buffer->data[2][(y/2) * decoding_frame_buffer->linesize[2] + x/2] ; - int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; - int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; - int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; + int B = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 1.596*(V - 128)))) ; + int G = std::min(255,std::max(0,(int)(1.164*(Y - 16) - 0.813*(V - 128) - 0.391*(U - 128)))) ; + int R = std::min(255,std::max(0,(int)(1.164*(Y - 16) + 2.018*(U - 128)))) ; - image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; - } - } - - if(len == decoding_buffer.size) - { - free(decoding_buffer.data) ; - decoding_buffer.data = NULL; - decoding_buffer.size = 0; - } - else if(len != 0) - { -#ifdef DEBUG_MPEG_VIDEO - std::cerr << "Moving remaining data (" << decoding_buffer.size - len << " bytes) back to 0" << std::endl; -#endif - - memmove(decoding_buffer.data,decoding_buffer.data+len,decoding_buffer.size - len) ; - decoding_buffer.size -= len ; + image.setPixel(QPoint(x,y),QRgb( 0xff000000 + (R << 16) + (G << 8) + B)) ; + } + } } + /* flush the decoder */ + decoding_buffer.data = NULL; + decoding_buffer.size = 0; + //avcodec_decode_video2(decoding_context,decoding_frame_buffer,&got_frame,&decoding_buffer) ; return true ; } diff --git a/plugins/VOIP/interface/rsVOIP.h b/plugins/VOIP/interface/rsVOIP.h index 25fd0cfbd..2e7e56eb4 100644 --- a/plugins/VOIP/interface/rsVOIP.h +++ b/plugins/VOIP/interface/rsVOIP.h @@ -51,15 +51,15 @@ class RsVOIPPongResult struct RsVOIPDataChunk { - typedef enum { RS_VOIP_DATA_TYPE_UNKNOWN = 0x00, - RS_VOIP_DATA_TYPE_AUDIO = 0x01, - RS_VOIP_DATA_TYPE_VIDEO = 0x02 } RsVOIPDataType ; + typedef enum { RS_VOIP_DATA_TYPE_UNKNOWN = 0x00, + RS_VOIP_DATA_TYPE_AUDIO = 0x01, + RS_VOIP_DATA_TYPE_VIDEO = 0x02 } RsVOIPDataType ; void *data ; // create/delete using malloc/free. uint32_t size ; RsVOIPDataType type ; // video or audio - - void clear() ; + + void clear() ; }; class RsVOIP