This adds screencast support on Linux. This is based on ffmpeg for the encoding & muxing (default format is MP4 with AAC audio and H264 video). The microphone sound is grabbed using QAudioInput. The encoder should be able to handle virtually any input format (it has been tested with a stereo input at different sampling rates with one format (16-bit signed) but it should work fine with any number of channels, sample format etc.). The only problems I have run into so far are that desktop recording is very slow (compared to OS X) and that the last few video frames are usually not included in the video. This may be due to GOPs not being complete, but that's just a wild guess.preferencesAboutTextFull
Craig Watson
8 years ago
commit
1ee405bcf1
@ -0,0 +1,591 @@ |
||||
/*
|
||||
* Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) |
||||
* |
||||
* This file is part of OpenBoard. |
||||
* |
||||
* OpenBoard is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU General Public License as published by |
||||
* the Free Software Foundation, version 3 of the License, |
||||
* with a specific linking exception for the OpenSSL project's |
||||
* "OpenSSL" library (or with modified versions of it that use the |
||||
* same license as the "OpenSSL" library). |
||||
* |
||||
* OpenBoard is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU General Public License |
||||
* along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/ |
||||
|
||||
#include "UBFFmpegVideoEncoder.h" |
||||
|
||||
//-------------------------------------------------------------------------
|
||||
// Utility functions
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
QString avErrorToQString(int errnum) |
||||
{ |
||||
char error[AV_ERROR_MAX_STRING_SIZE]; |
||||
av_make_error_string(error, AV_ERROR_MAX_STRING_SIZE, errnum); |
||||
|
||||
return QString(error); |
||||
} |
||||
|
||||
/**
|
||||
* @brief Write a given frame to the audio stream or, if a null frame is passed, flush the stream. |
||||
* |
||||
* @param frame An AVFrame to be written to the stream, or NULL to flush the stream |
||||
* @param packet A (reusable) packet, used to temporarily store frame data |
||||
* @param stream The stream to write to |
||||
* @param outputFormatContext The output format context |
||||
*/ |
||||
void writeFrame(AVFrame *frame, AVPacket *packet, AVStream *stream, AVFormatContext *outputFormatContext) |
||||
{ |
||||
int gotOutput, ret; |
||||
|
||||
av_init_packet(packet); |
||||
|
||||
do { |
||||
if (stream->codec->codec_type == AVMEDIA_TYPE_AUDIO) |
||||
ret = avcodec_encode_audio2(stream->codec, packet, frame, &gotOutput); |
||||
else |
||||
ret = avcodec_encode_video2(stream->codec, packet, frame, &gotOutput); |
||||
|
||||
if (ret < 0) |
||||
qWarning() << "Couldn't encode audio frame: " << avErrorToQString(ret); |
||||
|
||||
else if (gotOutput) { |
||||
AVRational codecTimebase = stream->codec->time_base; |
||||
AVRational streamVideoTimebase = stream->time_base; |
||||
|
||||
av_packet_rescale_ts(packet, codecTimebase, streamVideoTimebase); |
||||
packet->stream_index = stream->index; |
||||
|
||||
av_interleaved_write_frame(outputFormatContext, packet); |
||||
av_packet_unref(packet); |
||||
} |
||||
|
||||
} while (gotOutput && !frame); |
||||
} |
||||
|
||||
void flushStream(AVPacket *packet, AVStream *stream, AVFormatContext *outputFormatContext) |
||||
{ |
||||
writeFrame(NULL, packet, stream, outputFormatContext); |
||||
} |
||||
|
||||
//-------------------------------------------------------------------------
|
||||
// UBFFmpegVideoEncoder
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent) |
||||
: UBAbstractVideoEncoder(parent) |
||||
, mOutputFormatContext(NULL) |
||||
, mSwsContext(NULL) |
||||
, mShouldRecordAudio(true) |
||||
, mAudioInput(NULL) |
||||
, mSwrContext(NULL) |
||||
, mAudioOutBuffer(NULL) |
||||
, mAudioSampleRate(44100) |
||||
, mAudioFrameCount(0) |
||||
{ |
||||
|
||||
mVideoTimebase = 100 * framesPerSecond(); |
||||
|
||||
mVideoEncoderThread = new QThread; |
||||
mVideoWorker = new UBFFmpegVideoEncoderWorker(this); |
||||
mVideoWorker->moveToThread(mVideoEncoderThread); |
||||
|
||||
connect(mVideoWorker, SIGNAL(error(QString)), |
||||
this, SLOT(setLastErrorMessage(QString))); |
||||
|
||||
connect(mVideoEncoderThread, SIGNAL(started()), |
||||
mVideoWorker, SLOT(runEncoding())); |
||||
|
||||
connect(mVideoWorker, SIGNAL(encodingFinished()), |
||||
mVideoEncoderThread, SLOT(quit())); |
||||
|
||||
connect(mVideoEncoderThread, SIGNAL(finished()), |
||||
this, SLOT(finishEncoding())); |
||||
} |
||||
|
||||
UBFFmpegVideoEncoder::~UBFFmpegVideoEncoder() |
||||
{ |
||||
if (mVideoWorker) |
||||
delete mVideoWorker; |
||||
|
||||
if (mVideoEncoderThread) |
||||
delete mVideoEncoderThread; |
||||
|
||||
if (mAudioInput) |
||||
delete mAudioInput; |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage) |
||||
{ |
||||
qWarning() << "FFmpeg video encoder:" << pMessage; |
||||
mLastErrorMessage = pMessage; |
||||
} |
||||
|
||||
|
||||
bool UBFFmpegVideoEncoder::start() |
||||
{ |
||||
bool initialized = init(); |
||||
|
||||
if (initialized) { |
||||
mVideoEncoderThread->start(); |
||||
if (mShouldRecordAudio) |
||||
mAudioInput->start(); |
||||
} |
||||
|
||||
return initialized; |
||||
} |
||||
|
||||
bool UBFFmpegVideoEncoder::stop() |
||||
{ |
||||
qDebug() << "Video encoder: stop requested"; |
||||
|
||||
mVideoWorker->stopEncoding(); |
||||
|
||||
if (mShouldRecordAudio) |
||||
mAudioInput->stop(); |
||||
|
||||
return true; |
||||
} |
||||
|
||||
bool UBFFmpegVideoEncoder::init() |
||||
{ |
||||
av_register_all(); |
||||
avcodec_register_all(); |
||||
|
||||
AVDictionary * options = NULL; |
||||
int ret; |
||||
|
||||
// Output format and context
|
||||
// --------------------------------------
|
||||
if (avformat_alloc_output_context2(&mOutputFormatContext, NULL, |
||||
"mp4", NULL) < 0) |
||||
{ |
||||
setLastErrorMessage("Couldn't allocate video format context"); |
||||
return false; |
||||
} |
||||
|
||||
// The default codecs for mp4 are h264 and aac, we use those
|
||||
|
||||
|
||||
// Video codec and context
|
||||
// -------------------------------------
|
||||
mVideoStream = avformat_new_stream(mOutputFormatContext, 0); |
||||
|
||||
AVCodec * videoCodec = avcodec_find_encoder(mOutputFormatContext->oformat->video_codec); |
||||
if (!videoCodec) { |
||||
setLastErrorMessage("Video codec not found"); |
||||
return false; |
||||
} |
||||
|
||||
AVCodecContext* c = avcodec_alloc_context3(videoCodec); |
||||
|
||||
c->bit_rate = videoBitsPerSecond(); |
||||
c->width = videoSize().width(); |
||||
c->height = videoSize().height(); |
||||
c->time_base = {1, mVideoTimebase}; |
||||
c->gop_size = 10; |
||||
c->max_b_frames = 0; |
||||
c->pix_fmt = AV_PIX_FMT_YUV420P; |
||||
|
||||
if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER) |
||||
c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; |
||||
|
||||
/*
|
||||
* Supported pixel formats for h264 are: |
||||
* AV_PIX_FMT_YUV420P |
||||
* AV_PIX_FMT_YUV422P |
||||
* AV_PIX_FMT_YUV444P |
||||
* AV_PIX_FMT_YUVJ420P |
||||
*/ |
||||
|
||||
av_dict_set(&options, "preset", "slow", 0); |
||||
av_dict_set(&options, "crf", "20", 0); |
||||
|
||||
ret = avcodec_open2(c, videoCodec, &options); |
||||
|
||||
if (ret < 0) { |
||||
setLastErrorMessage(QString("Couldn't open video codec: ") + avErrorToQString(ret)); |
||||
return false; |
||||
} |
||||
|
||||
mVideoStream->codec = c; |
||||
|
||||
// Source images are RGB32, and should be converted to YUV for h264 video
|
||||
mSwsContext = sws_getCachedContext(mSwsContext, |
||||
c->width, c->height, AV_PIX_FMT_RGB32, |
||||
c->width, c->height, c->pix_fmt, |
||||
SWS_BICUBIC, 0, 0, 0); |
||||
|
||||
// Audio codec and context
|
||||
// -------------------------------------
|
||||
if (mShouldRecordAudio) { |
||||
|
||||
// Microphone input
|
||||
|
||||
mAudioInput = new UBMicrophoneInput(); |
||||
|
||||
connect(mAudioInput, SIGNAL(audioLevelChanged(quint8)), |
||||
this, SIGNAL(audioLevelChanged(quint8))); |
||||
|
||||
connect(mAudioInput, SIGNAL(dataAvailable(QByteArray)), |
||||
this, SLOT(onAudioAvailable(QByteArray))); |
||||
|
||||
if (!mAudioInput->init()) { |
||||
setLastErrorMessage("Couldn't initialize audio input"); |
||||
return false; |
||||
} |
||||
|
||||
int inChannelCount = mAudioInput->channelCount(); |
||||
int inSampleRate = mAudioInput->sampleRate(); |
||||
|
||||
// Codec
|
||||
|
||||
AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec); |
||||
|
||||
if (!audioCodec) { |
||||
setLastErrorMessage("Audio codec not found"); |
||||
return false; |
||||
} |
||||
|
||||
mAudioStream = avformat_new_stream(mOutputFormatContext, audioCodec); |
||||
mAudioStream->id = mOutputFormatContext->nb_streams-1; |
||||
|
||||
c = mAudioStream->codec; |
||||
|
||||
c->bit_rate = 96000; |
||||
c->sample_fmt = audioCodec->sample_fmts[0]; // FLTP by default for AAC
|
||||
c->sample_rate = mAudioSampleRate; |
||||
c->channels = 2; |
||||
c->channel_layout = av_get_default_channel_layout(c->channels); |
||||
c->profile = FF_PROFILE_AAC_MAIN; |
||||
c->time_base = {1, mAudioSampleRate}; |
||||
|
||||
if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER) |
||||
c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; |
||||
|
||||
ret = avcodec_open2(c, audioCodec, NULL); |
||||
|
||||
if (ret < 0) { |
||||
setLastErrorMessage(QString("Couldn't open audio codec: ") + avErrorToQString(ret)); |
||||
return false; |
||||
} |
||||
|
||||
// The input (raw sound from the microphone) may not match the codec's sampling rate,
|
||||
// sample format or number of channels; we use libswresample to convert and resample it
|
||||
mSwrContext = swr_alloc(); |
||||
if (!mSwrContext) { |
||||
setLastErrorMessage("Could not allocate resampler context"); |
||||
return false; |
||||
} |
||||
|
||||
av_opt_set_int(mSwrContext, "in_channel_count", inChannelCount, 0); |
||||
av_opt_set_int(mSwrContext, "in_sample_rate", inSampleRate, 0); |
||||
av_opt_set_sample_fmt(mSwrContext, "in_sample_fmt", (AVSampleFormat)mAudioInput->sampleFormat(), 0); |
||||
av_opt_set_int(mSwrContext, "out_channel_count", c->channels, 0); |
||||
av_opt_set_int(mSwrContext, "out_sample_rate", c->sample_rate, 0); |
||||
av_opt_set_sample_fmt(mSwrContext, "out_sample_fmt", c->sample_fmt, 0); |
||||
|
||||
ret = swr_init(mSwrContext); |
||||
if (ret < 0) { |
||||
setLastErrorMessage(QString("Couldn't initialize the resampling context: ") + avErrorToQString(ret)); |
||||
return false; |
||||
} |
||||
|
||||
// Buffer for resampled/converted audio
|
||||
mAudioOutBuffer = av_audio_fifo_alloc(c->sample_fmt, c->channels, c->frame_size); |
||||
} |
||||
|
||||
|
||||
// Open the output file
|
||||
ret = avio_open(&(mOutputFormatContext->pb), videoFileName().toStdString().c_str(), AVIO_FLAG_WRITE); |
||||
|
||||
if (ret < 0) { |
||||
setLastErrorMessage(QString("Couldn't open video file for writing: ") + avErrorToQString(ret)); |
||||
return false; |
||||
} |
||||
|
||||
// Write stream header
|
||||
ret = avformat_write_header(mOutputFormatContext, NULL); |
||||
|
||||
if (ret < 0) { |
||||
setLastErrorMessage(QString("Couldn't write header to file: ") + avErrorToQString(ret)); |
||||
return false; |
||||
} |
||||
|
||||
return true; |
||||
} |
||||
|
||||
/**
|
||||
* This function should be called every time a new "screenshot" is ready. |
||||
* The image is converted to the right format and sent to the encoder. |
||||
*/ |
||||
void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) |
||||
{ |
||||
if (!mVideoWorker->isRunning()) { |
||||
qDebug() << "Encoder worker thread not running. Queuing frame."; |
||||
mPendingFrames.enqueue({pImage, timestamp}); |
||||
} |
||||
|
||||
else { |
||||
// First send any queued frames, then the latest one
|
||||
while (!mPendingFrames.isEmpty()) { |
||||
AVFrame* avFrame = convertImageFrame(mPendingFrames.dequeue()); |
||||
if (avFrame) |
||||
mVideoWorker->queueVideoFrame(avFrame); |
||||
} |
||||
|
||||
// note: if converting the frame turns out to be too slow to do here, it
|
||||
// can always be done from the worker thread (in that case,
|
||||
// the worker's queue would contain ImageFrames rather than AVFrames)
|
||||
|
||||
AVFrame* avFrame = convertImageFrame({pImage, timestamp}); |
||||
if (avFrame) |
||||
mVideoWorker->queueVideoFrame(avFrame); |
||||
|
||||
// signal the worker that frames are available
|
||||
mVideoWorker->mWaitCondition.wakeAll(); |
||||
} |
||||
} |
||||
|
||||
/**
|
||||
* Convert a frame consisting of a QImage and timestamp to an AVFrame |
||||
* with the right pixel format and PTS |
||||
*/ |
||||
AVFrame* UBFFmpegVideoEncoder::convertImageFrame(ImageFrame frame) |
||||
{ |
||||
AVFrame* avFrame = av_frame_alloc(); |
||||
|
||||
avFrame->format = mVideoStream->codec->pix_fmt; |
||||
avFrame->width = mVideoStream->codec->width; |
||||
avFrame->height = mVideoStream->codec->height; |
||||
avFrame->pts = mVideoTimebase * frame.timestamp / 1000; |
||||
|
||||
const uchar * rgbImage = frame.image.bits(); |
||||
|
||||
const int in_linesize[1] = { frame.image.bytesPerLine() }; |
||||
|
||||
// Allocate the output image
|
||||
if (av_image_alloc(avFrame->data, avFrame->linesize, mVideoStream->codec->width, |
||||
mVideoStream->codec->height, mVideoStream->codec->pix_fmt, 32) < 0) |
||||
{ |
||||
qWarning() << "Couldn't allocate image"; |
||||
return NULL; |
||||
} |
||||
|
||||
sws_scale(mSwsContext, |
||||
(const uint8_t* const*)&rgbImage, |
||||
in_linesize, |
||||
0, |
||||
mVideoStream->codec->height, |
||||
avFrame->data, |
||||
avFrame->linesize); |
||||
|
||||
return avFrame; |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoder::onAudioAvailable(QByteArray data) |
||||
{ |
||||
if (!data.isEmpty()) |
||||
processAudio(data); |
||||
} |
||||
|
||||
/**
|
||||
* Resample and convert audio to match the encoder's settings and queue the |
||||
* output. If enough output data is available, it is packaged into AVFrames and |
||||
* sent to the encoder thread. |
||||
*/ |
||||
void UBFFmpegVideoEncoder::processAudio(QByteArray &data) |
||||
{ |
||||
int ret; |
||||
AVCodecContext* codecContext = mAudioStream->codec; |
||||
|
||||
const char * inSamples = data.constData(); |
||||
|
||||
// The number of samples (per channel) in the input
|
||||
int inSamplesCount = data.size() / ((mAudioInput->sampleSize() / 8) * mAudioInput->channelCount()); |
||||
|
||||
// The number of samples we will get after conversion
|
||||
int outSamplesCount = swr_get_out_samples(mSwrContext, inSamplesCount); |
||||
|
||||
// Allocate output samples
|
||||
uint8_t ** outSamples = NULL; |
||||
int outSamplesLineSize; |
||||
|
||||
ret = av_samples_alloc_array_and_samples(&outSamples, &outSamplesLineSize, |
||||
codecContext->channels, outSamplesCount, |
||||
codecContext->sample_fmt, 0); |
||||
if (ret < 0) { |
||||
qWarning() << "Could not allocate audio samples" << avErrorToQString(ret); |
||||
return; |
||||
} |
||||
|
||||
// Convert to destination format
|
||||
ret = swr_convert(mSwrContext, |
||||
outSamples, outSamplesCount, |
||||
(const uint8_t **)&inSamples, inSamplesCount); |
||||
if (ret < 0) { |
||||
qWarning() << "Error converting audio samples: " << avErrorToQString(ret); |
||||
return; |
||||
} |
||||
|
||||
// Append the converted samples to the out buffer.
|
||||
ret = av_audio_fifo_write(mAudioOutBuffer, (void**)outSamples, outSamplesCount); |
||||
if (ret < 0) { |
||||
qWarning() << "Could not write to FIFO queue: " << avErrorToQString(ret); |
||||
return; |
||||
} |
||||
|
||||
// Keep the data queued until next call if the encoder thread isn't running
|
||||
if (!mVideoWorker->isRunning()) |
||||
return; |
||||
|
||||
bool framesAdded = false; |
||||
while (av_audio_fifo_size(mAudioOutBuffer) > codecContext->frame_size) { |
||||
AVFrame * avFrame = av_frame_alloc(); |
||||
avFrame->nb_samples = codecContext->frame_size; |
||||
avFrame->channel_layout = codecContext->channel_layout; |
||||
avFrame->format = codecContext->sample_fmt; |
||||
avFrame->sample_rate = codecContext->sample_rate; |
||||
avFrame->pts = mAudioFrameCount; |
||||
|
||||
ret = av_frame_get_buffer(avFrame, 0); |
||||
if (ret < 0) { |
||||
qWarning() << "Couldn't allocate frame: " << avErrorToQString(ret); |
||||
break; |
||||
} |
||||
|
||||
ret = av_audio_fifo_read(mAudioOutBuffer, (void**)avFrame->data, codecContext->frame_size); |
||||
if (ret < 0) |
||||
qWarning() << "Could not read from FIFO queue: " << avErrorToQString(ret); |
||||
|
||||
else { |
||||
mAudioFrameCount += codecContext->frame_size; |
||||
|
||||
mVideoWorker->queueAudioFrame(avFrame); |
||||
framesAdded = true; |
||||
} |
||||
} |
||||
|
||||
if (framesAdded) |
||||
mVideoWorker->mWaitCondition.wakeAll(); |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoder::finishEncoding() |
||||
{ |
||||
qDebug() << "VideoEncoder::finishEncoding called"; |
||||
|
||||
flushStream(mVideoWorker->mVideoPacket, mVideoStream, mOutputFormatContext); |
||||
|
||||
if (mShouldRecordAudio) |
||||
flushStream(mVideoWorker->mAudioPacket, mAudioStream, mOutputFormatContext); |
||||
|
||||
av_write_trailer(mOutputFormatContext); |
||||
avio_close(mOutputFormatContext->pb); |
||||
|
||||
avcodec_close(mVideoStream->codec); |
||||
sws_freeContext(mSwsContext); |
||||
|
||||
if (mShouldRecordAudio) { |
||||
avcodec_close(mAudioStream->codec); |
||||
swr_free(&mSwrContext); |
||||
} |
||||
|
||||
avformat_free_context(mOutputFormatContext); |
||||
|
||||
emit encodingFinished(true); |
||||
} |
||||
|
||||
|
||||
//-------------------------------------------------------------------------
|
||||
// Worker
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
UBFFmpegVideoEncoderWorker::UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller) |
||||
: mController(controller) |
||||
{ |
||||
mStopRequested = false; |
||||
mIsRunning = false; |
||||
mVideoPacket = new AVPacket(); |
||||
mAudioPacket = new AVPacket(); |
||||
} |
||||
|
||||
UBFFmpegVideoEncoderWorker::~UBFFmpegVideoEncoderWorker() |
||||
{ |
||||
if (mVideoPacket) |
||||
delete mVideoPacket; |
||||
|
||||
if (mAudioPacket) |
||||
delete mAudioPacket; |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoderWorker::stopEncoding() |
||||
{ |
||||
qDebug() << "Video worker: stop requested"; |
||||
mStopRequested = true; |
||||
mWaitCondition.wakeAll(); |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoderWorker::queueVideoFrame(AVFrame* frame) |
||||
{ |
||||
if (frame) { |
||||
mFrameQueueMutex.lock(); |
||||
mImageQueue.enqueue(frame); |
||||
mFrameQueueMutex.unlock(); |
||||
} |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoderWorker::queueAudioFrame(AVFrame* frame) |
||||
{ |
||||
if (frame) { |
||||
mFrameQueueMutex.lock(); |
||||
mAudioQueue.enqueue(frame); |
||||
mFrameQueueMutex.unlock(); |
||||
} |
||||
} |
||||
|
||||
/**
|
||||
* The main encoding function. Takes the queued frames and |
||||
* writes them to the video and audio streams |
||||
*/ |
||||
void UBFFmpegVideoEncoderWorker::runEncoding() |
||||
{ |
||||
mIsRunning = true; |
||||
|
||||
while (!mStopRequested) { |
||||
mFrameQueueMutex.lock(); |
||||
mWaitCondition.wait(&mFrameQueueMutex); |
||||
|
||||
while (!mImageQueue.isEmpty()) { |
||||
writeLatestVideoFrame(); |
||||
} |
||||
|
||||
while (!mAudioQueue.isEmpty()) { |
||||
writeLatestAudioFrame(); |
||||
} |
||||
|
||||
mFrameQueueMutex.unlock(); |
||||
} |
||||
|
||||
emit encodingFinished(); |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame() |
||||
{ |
||||
AVFrame* frame = mImageQueue.dequeue(); |
||||
writeFrame(frame, mVideoPacket, mController->mVideoStream, mController->mOutputFormatContext); |
||||
av_frame_free(&frame); |
||||
} |
||||
|
||||
void UBFFmpegVideoEncoderWorker::writeLatestAudioFrame() |
||||
{ |
||||
AVFrame *frame = mAudioQueue.dequeue(); |
||||
writeFrame(frame, mAudioPacket, mController->mAudioStream, mController->mOutputFormatContext); |
||||
av_frame_free(&frame); |
||||
} |
||||
@ -0,0 +1,183 @@ |
||||
/*
|
||||
* Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) |
||||
* |
||||
* This file is part of OpenBoard. |
||||
* |
||||
* OpenBoard is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU General Public License as published by |
||||
* the Free Software Foundation, version 3 of the License, |
||||
* with a specific linking exception for the OpenSSL project's |
||||
* "OpenSSL" library (or with modified versions of it that use the |
||||
* same license as the "OpenSSL" library). |
||||
* |
||||
* OpenBoard is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU General Public License |
||||
* along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/ |
||||
|
||||
#ifndef UBFFMPEGVIDEOENCODER_H |
||||
#define UBFFMPEGVIDEOENCODER_H |
||||
|
||||
extern "C" { |
||||
#include <libavcodec/avcodec.h> |
||||
#include <libavformat/avformat.h> |
||||
#include <libavformat/avio.h> |
||||
#include <libavutil/audio_fifo.h> |
||||
#include <libavutil/avutil.h> |
||||
#include <libavutil/imgutils.h> |
||||
#include <libavutil/opt.h> |
||||
#include <libavutil/mathematics.h> |
||||
#include <libavutil/time.h> |
||||
#include <libswscale/swscale.h> |
||||
#include <libswresample/swresample.h> |
||||
} |
||||
|
||||
#include <atomic> |
||||
|
||||
#include <QtCore> |
||||
#include <QImage> |
||||
|
||||
#include "podcast/UBAbstractVideoEncoder.h" |
||||
#include "podcast/ffmpeg/UBMicrophoneInput.h" |
||||
|
||||
class UBFFmpegVideoEncoderWorker; |
||||
class UBPodcastController; |
||||
|
||||
/**
|
||||
* This class provides an interface between the podcast controller and the ffmpeg |
||||
* back-end. |
||||
* It includes all the necessary objects and methods to record video (muxer, audio and |
||||
* video streams and encoders, etc) from inputs consisting of raw PCM audio and raw RGBA |
||||
* images. |
||||
* |
||||
* A worker thread is used to encode and write the audio and video on-the-fly. |
||||
*/ |
||||
|
||||
class UBFFmpegVideoEncoder : public UBAbstractVideoEncoder |
||||
{ |
||||
Q_OBJECT |
||||
|
||||
friend class UBFFmpegVideoEncoderWorker; |
||||
|
||||
public: |
||||
|
||||
UBFFmpegVideoEncoder(QObject* parent = NULL); |
||||
virtual ~UBFFmpegVideoEncoder(); |
||||
|
||||
bool start(); |
||||
bool stop(); |
||||
|
||||
void newPixmap(const QImage& pImage, long timestamp); |
||||
|
||||
QString videoFileExtension() const { return "mp4"; } |
||||
|
||||
QString lastErrorMessage() { return mLastErrorMessage; } |
||||
|
||||
void setRecordAudio(bool pRecordAudio) { mShouldRecordAudio = pRecordAudio; } |
||||
|
||||
signals: |
||||
|
||||
void encodingFinished(bool ok); |
||||
|
||||
private slots: |
||||
|
||||
void setLastErrorMessage(const QString& pMessage); |
||||
void onAudioAvailable(QByteArray data); |
||||
void finishEncoding(); |
||||
|
||||
private: |
||||
|
||||
struct ImageFrame |
||||
{ |
||||
QImage image; |
||||
long timestamp; // unit: ms
|
||||
}; |
||||
|
||||
AVFrame* convertImageFrame(ImageFrame frame); |
||||
AVFrame* convertAudio(QByteArray data); |
||||
void processAudio(QByteArray& data); |
||||
bool init(); |
||||
|
||||
QString mLastErrorMessage; |
||||
|
||||
QThread* mVideoEncoderThread; |
||||
UBFFmpegVideoEncoderWorker* mVideoWorker; |
||||
|
||||
// Muxer
|
||||
// ------------------------------------------
|
||||
AVFormatContext* mOutputFormatContext; |
||||
AVStream* mVideoStream; |
||||
AVStream* mAudioStream; |
||||
|
||||
// Video
|
||||
// ------------------------------------------
|
||||
QQueue<ImageFrame> mPendingFrames; |
||||
struct SwsContext * mSwsContext; |
||||
|
||||
int mVideoTimebase; |
||||
|
||||
// Audio
|
||||
// ------------------------------------------
|
||||
bool mShouldRecordAudio; |
||||
|
||||
UBMicrophoneInput * mAudioInput; |
||||
struct SwrContext * mSwrContext; |
||||
/// Queue for audio that has been rescaled/converted but not encoded yet
|
||||
AVAudioFifo *mAudioOutBuffer; |
||||
|
||||
/// Sample rate for encoded audio
|
||||
int mAudioSampleRate; |
||||
/// Total audio frames sent to encoder
|
||||
int mAudioFrameCount; |
||||
}; |
||||
|
||||
|
||||
class UBFFmpegVideoEncoderWorker : public QObject |
||||
{ |
||||
Q_OBJECT |
||||
|
||||
friend class UBFFmpegVideoEncoder; |
||||
|
||||
public: |
||||
UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller); |
||||
~UBFFmpegVideoEncoderWorker(); |
||||
|
||||
bool isRunning() { return mIsRunning; } |
||||
|
||||
void queueVideoFrame(AVFrame* frame); |
||||
void queueAudioFrame(AVFrame* frame); |
||||
|
||||
public slots: |
||||
void runEncoding(); |
||||
void stopEncoding(); |
||||
|
||||
signals: |
||||
void encodingFinished(); |
||||
void error(QString message); |
||||
|
||||
private: |
||||
void writeLatestVideoFrame(); |
||||
void writeLatestAudioFrame(); |
||||
|
||||
UBFFmpegVideoEncoder* mController; |
||||
|
||||
// std::atomic is C++11. This won't work with msvc2010, so a
|
||||
// newer compiler must be used if this class is to be used on Windows
|
||||
std::atomic<bool> mStopRequested; |
||||
std::atomic<bool> mIsRunning; |
||||
|
||||
QQueue<AVFrame*> mImageQueue; |
||||
QQueue<AVFrame*> mAudioQueue; |
||||
|
||||
QMutex mFrameQueueMutex; |
||||
QWaitCondition mWaitCondition; |
||||
|
||||
AVPacket* mVideoPacket; |
||||
AVPacket* mAudioPacket; |
||||
}; |
||||
|
||||
#endif // UBFFMPEGVIDEOENCODER_H
|
||||
@ -0,0 +1,299 @@ |
||||
/*
|
||||
* Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) |
||||
* |
||||
* This file is part of OpenBoard. |
||||
* |
||||
* OpenBoard is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU General Public License as published by |
||||
* the Free Software Foundation, version 3 of the License, |
||||
* with a specific linking exception for the OpenSSL project's |
||||
* "OpenSSL" library (or with modified versions of it that use the |
||||
* same license as the "OpenSSL" library). |
||||
* |
||||
* OpenBoard is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU General Public License |
||||
* along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/ |
||||
|
||||
#include "UBMicrophoneInput.h" |
||||
|
||||
UBMicrophoneInput::UBMicrophoneInput() |
||||
: mAudioInput(NULL) |
||||
, mIODevice(NULL) |
||||
, mSeekPos(0) |
||||
{ |
||||
} |
||||
|
||||
UBMicrophoneInput::~UBMicrophoneInput() |
||||
{ |
||||
if (mAudioInput) |
||||
delete mAudioInput; |
||||
} |
||||
|
||||
bool UBMicrophoneInput::init() |
||||
{ |
||||
if (mAudioDeviceInfo.isNull()) { |
||||
qWarning("No audio input device selected; using default"); |
||||
mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice(); |
||||
} |
||||
|
||||
mAudioFormat = mAudioDeviceInfo.preferredFormat(); |
||||
mAudioInput = new QAudioInput(mAudioDeviceInfo, mAudioFormat, NULL); |
||||
|
||||
connect(mAudioInput, SIGNAL(stateChanged(QAudio::State)), |
||||
this, SLOT(onAudioInputStateChanged(QAudio::State))); |
||||
|
||||
|
||||
qDebug() << "Input device name: " << mAudioDeviceInfo.deviceName(); |
||||
qDebug() << "Input sample format: " << mAudioFormat.sampleSize() << "bit" |
||||
<< mAudioFormat.sampleType() << "at" << mAudioFormat.sampleRate() << "Hz" |
||||
<< "; codec: " << mAudioFormat.codec(); |
||||
|
||||
return true; |
||||
} |
||||
|
||||
void UBMicrophoneInput::start() |
||||
{ |
||||
mIODevice = mAudioInput->start(); |
||||
|
||||
connect(mIODevice, SIGNAL(readyRead()), |
||||
this, SLOT(onDataReady())); |
||||
|
||||
if (mAudioInput->error() == QAudio::OpenError) |
||||
qWarning() << "Error opening audio input"; |
||||
} |
||||
|
||||
void UBMicrophoneInput::stop() |
||||
{ |
||||
mAudioInput->stop(); |
||||
} |
||||
|
||||
QStringList UBMicrophoneInput::availableDevicesNames() |
||||
{ |
||||
QStringList names; |
||||
QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput); |
||||
|
||||
foreach (QAudioDeviceInfo device, devices) { |
||||
names.push_back(device.deviceName()); |
||||
} |
||||
|
||||
return names; |
||||
} |
||||
|
||||
void UBMicrophoneInput::setInputDevice(QString name) |
||||
{ |
||||
if (name.isEmpty()) { |
||||
mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice(); |
||||
return; |
||||
} |
||||
|
||||
QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput); |
||||
bool found = false; |
||||
|
||||
foreach (QAudioDeviceInfo device, devices) { |
||||
if (device.deviceName() == name) { |
||||
mAudioDeviceInfo = device; |
||||
found = true; |
||||
break; |
||||
} |
||||
} |
||||
|
||||
if (!found) { |
||||
qWarning() << "Audio input device not found; using default instead"; |
||||
mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice(); |
||||
} |
||||
|
||||
} |
||||
|
||||
int UBMicrophoneInput::channelCount() |
||||
{ |
||||
return mAudioFormat.channelCount(); |
||||
} |
||||
|
||||
int UBMicrophoneInput::sampleRate() |
||||
{ |
||||
return mAudioFormat.sampleRate(); |
||||
} |
||||
|
||||
/* Return the sample size in bits */ |
||||
int UBMicrophoneInput::sampleSize() |
||||
{ |
||||
return mAudioFormat.sampleSize(); |
||||
} |
||||
|
||||
/** Return the sample format in FFMpeg style (AVSampleFormat enum) */ |
||||
int UBMicrophoneInput::sampleFormat() |
||||
{ |
||||
enum AVSampleFormat { |
||||
AV_SAMPLE_FMT_NONE = -1, |
||||
AV_SAMPLE_FMT_U8, |
||||
AV_SAMPLE_FMT_S16, |
||||
AV_SAMPLE_FMT_S32, |
||||
AV_SAMPLE_FMT_FLT, |
||||
AV_SAMPLE_FMT_DBL, |
||||
AV_SAMPLE_FMT_U8P, |
||||
AV_SAMPLE_FMT_S16P, |
||||
AV_SAMPLE_FMT_S32P, |
||||
AV_SAMPLE_FMT_FLTP, |
||||
AV_SAMPLE_FMT_DBLP, |
||||
AV_SAMPLE_FMT_NB |
||||
}; |
||||
|
||||
int sampleSize = mAudioFormat.sampleSize(); |
||||
QAudioFormat::SampleType sampleType = mAudioFormat.sampleType(); |
||||
|
||||
switch (sampleType) { |
||||
case QAudioFormat::Unknown: |
||||
return AV_SAMPLE_FMT_NONE; |
||||
|
||||
case QAudioFormat::SignedInt: |
||||
if (sampleSize == 16) |
||||
return AV_SAMPLE_FMT_S16; |
||||
if (sampleSize == 32) |
||||
return AV_SAMPLE_FMT_S32; |
||||
break; |
||||
|
||||
case QAudioFormat::UnSignedInt: |
||||
if (sampleSize == 8) |
||||
return AV_SAMPLE_FMT_U8; |
||||
break; |
||||
|
||||
case QAudioFormat::Float: |
||||
return AV_SAMPLE_FMT_FLT; |
||||
|
||||
default: |
||||
return AV_SAMPLE_FMT_NONE; |
||||
} |
||||
|
||||
return AV_SAMPLE_FMT_NONE; |
||||
} |
||||
|
||||
QString UBMicrophoneInput::codec() |
||||
{ |
||||
return mAudioFormat.codec(); |
||||
} |
||||
|
||||
static qint64 uSecsElapsed = 0; |
||||
void UBMicrophoneInput::onDataReady() |
||||
{ |
||||
int numBytes = mAudioInput->bytesReady(); |
||||
|
||||
uSecsElapsed += mAudioFormat.durationForBytes(numBytes); |
||||
|
||||
// Only emit data every 100ms
|
||||
if (uSecsElapsed > 100000) { |
||||
uSecsElapsed = 0; |
||||
QByteArray data = mIODevice->read(numBytes); |
||||
|
||||
quint8 level = audioLevel(data); |
||||
if (level != mLastAudioLevel) { |
||||
mLastAudioLevel = level; |
||||
emit audioLevelChanged(level); |
||||
} |
||||
|
||||
emit dataAvailable(data); |
||||
} |
||||
} |
||||
|
||||
void UBMicrophoneInput::onAudioInputStateChanged(QAudio::State state) |
||||
{ |
||||
switch (state) { |
||||
case QAudio::StoppedState: |
||||
if (mAudioInput->error() != QAudio::NoError) { |
||||
emit error(getErrorString(mAudioInput->error())); |
||||
} |
||||
break; |
||||
|
||||
// handle other states?
|
||||
|
||||
default: |
||||
break; |
||||
} |
||||
} |
||||
|
||||
/**
|
||||
* @brief Calculate the current audio level of an array of samples and return it |
||||
* @param data An array of audio samples |
||||
* @return A value between 0 and 255 |
||||
* |
||||
* Audio level is calculated as the RMS (root mean square) of the samples |
||||
* in the supplied array. |
||||
*/ |
||||
quint8 UBMicrophoneInput::audioLevel(const QByteArray &data) |
||||
{ |
||||
int bytesPerSample = mAudioFormat.bytesPerFrame() / mAudioFormat.channelCount(); |
||||
|
||||
const char * ptr = data.constData(); |
||||
double sum = 0; |
||||
int n_samples = data.size() / bytesPerSample; |
||||
|
||||
for (int i(0); i < (data.size() - bytesPerSample); i += bytesPerSample) { |
||||
sum += pow(sampleRelativeLevel(ptr + i), 2); |
||||
} |
||||
|
||||
double rms = sqrt(sum/n_samples); |
||||
|
||||
// The vu meter looks a bit better when the RMS isn't displayed linearly, as perceived sound
|
||||
// level increases logarithmically. So here RMS is substituted by rms^(1/e)
|
||||
rms = pow(rms, 1./exp(1)); |
||||
|
||||
return UINT8_MAX * rms; |
||||
} |
||||
|
||||
/**
|
||||
* @brief Calculate one sample's level relative to its maximum value |
||||
* @param sample One sample, in the format specified by mAudioFormat |
||||
* @return A double between 0 and 1.0, where 1.0 is the maximum value the sample can take, |
||||
* or -1 if the value couldn't be calculated. |
||||
*/ |
||||
double UBMicrophoneInput::sampleRelativeLevel(const char* sample) |
||||
{ |
||||
QAudioFormat::SampleType type = mAudioFormat.sampleType(); |
||||
int sampleSize = mAudioFormat.sampleSize(); |
||||
|
||||
if (sampleSize == 16 && type == QAudioFormat::SignedInt) |
||||
return double(*reinterpret_cast<const int16_t*>(sample))/INT16_MAX; |
||||
|
||||
if (sampleSize == 8 && type == QAudioFormat::SignedInt) |
||||
return double(*reinterpret_cast<const int8_t*>(sample))/INT8_MAX; |
||||
|
||||
if (sampleSize == 16 && type == QAudioFormat::UnSignedInt) |
||||
return double(*reinterpret_cast<const uint16_t*>(sample))/UINT16_MAX; |
||||
|
||||
if (sampleSize == 8 && type == QAudioFormat::UnSignedInt) |
||||
return double(*reinterpret_cast<const uint8_t*>(sample))/UINT8_MAX; |
||||
|
||||
if (type == QAudioFormat::Float) |
||||
return (*reinterpret_cast<const float*>(sample) + 1.0)/2.; |
||||
|
||||
return -1; |
||||
} |
||||
|
||||
/**
|
||||
* @brief Return a meaningful error string based on QAudio error codes |
||||
*/ |
||||
QString UBMicrophoneInput::getErrorString(QAudio::Error errorCode) |
||||
{ |
||||
switch (errorCode) { |
||||
case QAudio::NoError : |
||||
return ""; |
||||
|
||||
case QAudio::OpenError : |
||||
return "Couldn't open the audio device"; |
||||
|
||||
case QAudio::IOError : |
||||
return "Error reading from audio device"; |
||||
|
||||
case QAudio::UnderrunError : |
||||
return "Underrun error"; |
||||
|
||||
case QAudio::FatalError : |
||||
return "Fatal error; audio device unusable"; |
||||
|
||||
} |
||||
return ""; |
||||
} |
||||
@ -0,0 +1,81 @@ |
||||
/*
|
||||
* Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) |
||||
* |
||||
* This file is part of OpenBoard. |
||||
* |
||||
* OpenBoard is free software: you can redistribute it and/or modify |
||||
* it under the terms of the GNU General Public License as published by |
||||
* the Free Software Foundation, version 3 of the License, |
||||
* with a specific linking exception for the OpenSSL project's |
||||
* "OpenSSL" library (or with modified versions of it that use the |
||||
* same license as the "OpenSSL" library). |
||||
* |
||||
* OpenBoard is distributed in the hope that it will be useful, |
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of |
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
||||
* GNU General Public License for more details. |
||||
* |
||||
* You should have received a copy of the GNU General Public License |
||||
* along with OpenBoard. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/ |
||||
|
||||
#ifndef UBMICROPHONEINPUT_H |
||||
#define UBMICROPHONEINPUT_H |
||||
|
||||
#include <QtCore> |
||||
#include <QAudioInput> |
||||
|
||||
/**
|
||||
* @brief The UBMicrophoneInput class captures uncompressed sound from a microphone. |
||||
* |
||||
* Audio samples can be read by connecting to the dataAvailable signal. |
||||
*/ |
||||
class UBMicrophoneInput : public QObject |
||||
{ |
||||
Q_OBJECT |
||||
|
||||
public: |
||||
UBMicrophoneInput(); |
||||
virtual ~UBMicrophoneInput(); |
||||
|
||||
bool init(); |
||||
void start(); |
||||
void stop(); |
||||
|
||||
static QStringList availableDevicesNames(); |
||||
void setInputDevice(QString name = ""); |
||||
|
||||
int channelCount(); |
||||
int sampleRate(); |
||||
int sampleSize(); |
||||
int sampleFormat(); |
||||
QString codec(); |
||||
|
||||
signals: |
||||
/// Send the new audio level, between 0 and 255
|
||||
void audioLevelChanged(quint8 level); |
||||
|
||||
/// Emitted when new audio data is available
|
||||
void dataAvailable(QByteArray data); |
||||
|
||||
void error(QString message); |
||||
|
||||
private slots: |
||||
void onAudioInputStateChanged(QAudio::State state); |
||||
void onDataReady(); |
||||
|
||||
private: |
||||
double sampleRelativeLevel(const char* sample); |
||||
quint8 audioLevel(const QByteArray& data); |
||||
QString getErrorString(QAudio::Error errorCode); |
||||
|
||||
QAudioInput* mAudioInput; |
||||
QIODevice * mIODevice; |
||||
QAudioDeviceInfo mAudioDeviceInfo; |
||||
QAudioFormat mAudioFormat; |
||||
|
||||
qint64 mSeekPos; |
||||
quint8 mLastAudioLevel; |
||||
}; |
||||
|
||||
#endif // UBMICROPHONEINPUT_H
|
||||
Loading…
Reference in new issue