From a2fb735bbc4938e6f2a0a6681a987511587c57fb Mon Sep 17 00:00:00 2001 From: Craig Watson Date: Thu, 28 Apr 2016 15:58:52 +0200 Subject: [PATCH 1/5] Podcasts on Linux: working video, no audio yet --- src/core/UBApplication.cpp | 2 - src/podcast/UBPodcastController.cpp | 4 + src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp | 396 ++++++++++++++++++++ src/podcast/ffmpeg/UBFFmpegVideoEncoder.h | 135 +++++++ src/podcast/podcast.pri | 25 ++ 5 files changed, 560 insertions(+), 2 deletions(-) create mode 100644 src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp create mode 100644 src/podcast/ffmpeg/UBFFmpegVideoEncoder.h diff --git a/src/core/UBApplication.cpp b/src/core/UBApplication.cpp index e364027d..5e205fea 100644 --- a/src/core/UBApplication.cpp +++ b/src/core/UBApplication.cpp @@ -520,10 +520,8 @@ void UBApplication::decorateActionMenu(QAction* action) menu->addSeparator(); -#ifndef Q_OS_LINUX // No Podcast on Linux yet menu->addAction(mainWindow->actionPodcast); mainWindow->actionPodcast->setText(tr("Podcast")); -#endif menu->addSeparator(); menu->addAction(mainWindow->actionQuit); diff --git a/src/podcast/UBPodcastController.cpp b/src/podcast/UBPodcastController.cpp index 598b5135..20a96300 100644 --- a/src/podcast/UBPodcastController.cpp +++ b/src/podcast/UBPodcastController.cpp @@ -64,6 +64,8 @@ #elif defined(Q_OS_OSX) #include "quicktime/UBQuickTimeVideoEncoder.h" #include "quicktime/UBAudioQueueRecorder.h" +#elif defined(Q_OS_LINUX) + #include "ffmpeg/UBFFmpegVideoEncoder.h" #endif #include "core/memcheck.h" @@ -309,6 +311,8 @@ void UBPodcastController::start() mVideoEncoder = new UBWindowsMediaVideoEncoder(this); //deleted on stop #elif defined(Q_OS_OSX) mVideoEncoder = new UBQuickTimeVideoEncoder(this); //deleted on stop +#elif defined(Q_OS_LINUX) + mVideoEncoder = new UBFFmpegVideoEncoder(this); #endif if (mVideoEncoder) diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp new file mode 100644 index 00000000..d2dd361a --- /dev/null +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp @@ -0,0 +1,396 @@ +#include "UBFFmpegVideoEncoder.h" + +// Future proofing +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1) +#define av_frame_alloc avcodec_alloc_frame +#define av_frame_free avcodec_free_frame +#endif + +QString avErrorToQString(int errnum) +{ + char error[AV_ERROR_MAX_STRING_SIZE]; + av_make_error_string(error, AV_ERROR_MAX_STRING_SIZE, errnum); + + return QString(error); +} + +/** + * @brief Constructor for the ffmpeg video encoder + * + * + * This class provides an interface between the screencast controller and the ffmpeg + * back-end. It initializes the audio and video encoders and frees them when done; + * worker threads handle the actual encoding of frames. + * + */ +UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent) + : UBAbstractVideoEncoder(parent) + , mOutputFormatContext(NULL) + , mSwsContext(NULL) + , mFile(NULL) +{ + + mTimebase = 100 * framesPerSecond(); + qDebug() << "timebase: " << mTimebase; + + mVideoEncoderThread = new QThread; + mVideoWorker = new UBFFmpegVideoEncoderWorker(this); + mVideoWorker->moveToThread(mVideoEncoderThread); + + connect(mVideoWorker, SIGNAL(error(QString)), + this, SLOT(setLastErrorMessage(QString))); + + connect(mVideoEncoderThread, SIGNAL(started()), + mVideoWorker, SLOT(runEncoding())); + + connect(mVideoWorker, SIGNAL(encodingFinished()), + mVideoEncoderThread, SLOT(quit())); + + connect(mVideoEncoderThread, SIGNAL(finished()), + this, SLOT(finishEncoding())); +} + +UBFFmpegVideoEncoder::~UBFFmpegVideoEncoder() +{ + if (mVideoWorker) + delete mVideoWorker; + + if (mVideoEncoderThread) + delete mVideoEncoderThread; + +} + +void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage) +{ + qDebug() << "FFmpeg video encoder:" << pMessage; + mLastErrorMessage = pMessage; +} + +bool UBFFmpegVideoEncoder::start() +{ + bool initialized = init(); + + if (initialized) + mVideoEncoderThread->start(); + + return initialized; +} + +bool UBFFmpegVideoEncoder::stop() +{ + qDebug() << "Video encoder: stop requested"; + + mVideoWorker->stopEncoding(); + + return true; +} + +bool UBFFmpegVideoEncoder::init() +{ + // Initialize ffmpeg lib + av_register_all(); + avcodec_register_all(); + + AVDictionary * options = NULL; + int ret; + + // Output format and context + // -------------------------------------- + + if (avformat_alloc_output_context2(&mOutputFormatContext, NULL, + "mp4", NULL) < 0) + { + setLastErrorMessage("Couldn't allocate video format context"); + return false; + } + + // The default codecs for mp4 are h264 and aac, we use those + + + // Video codec and context + // ------------------------------------- + + AVCodec * videoCodec = avcodec_find_encoder(mOutputFormatContext->oformat->video_codec); + if (!videoCodec) { + setLastErrorMessage("Video codec not found"); + return false; + } + + mVideoStream = avformat_new_stream(mOutputFormatContext, 0); + mVideoStream->time_base = {1, mTimebase}; + + avcodec_get_context_defaults3(mVideoStream->codec, videoCodec); + AVCodecContext* c = avcodec_alloc_context3(videoCodec); + + c->bit_rate = videoBitsPerSecond(); + c->width = videoSize().width(); + c->height = videoSize().height(); + c->time_base = {1, mTimebase}; + c->gop_size = 10; + c->max_b_frames = 0; + c->pix_fmt = AV_PIX_FMT_YUV420P; + + if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER) + c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + + /* + * Supported pixel formats for h264 are: + * AV_PIX_FMT_YUV420P + * AV_PIX_FMT_YUV422P + * AV_PIX_FMT_YUV444P + * AV_PIX_FMT_YUVJ420P + */ + + av_dict_set(&options, "preset", "slow", 0); + av_dict_set(&options, "crf", "20", 0); + + ret = avcodec_open2(c, videoCodec, &options); + + if (ret < 0) { + setLastErrorMessage(QString("Couldn't open video codec: ") + avErrorToQString(ret)); + return false; + } + + mVideoStream->codec = c; + + // Source images are RGB32, and should be converted to YUV for h264 video + mSwsContext = sws_getCachedContext(mSwsContext, + c->width, c->height, AV_PIX_FMT_RGB32, + c->width, c->height, c->pix_fmt, + SWS_BICUBIC, 0, 0, 0); + + // Audio codec and context + // ------------------------------------- + /* + AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec); + mAudioStream = avformat_new_stream(mOutputFormatContext, audioCodec); + */ + + + // Open the output file + ret = avio_open(&(mOutputFormatContext->pb), videoFileName().toStdString().c_str(), AVIO_FLAG_WRITE); + + if (ret < 0) { + setLastErrorMessage(QString("Couldn't open video file for writing: ") + avErrorToQString(ret)); + return false; + } + + // Write stream header + ret = avformat_write_header(mOutputFormatContext, NULL); + + if (ret < 0) { + setLastErrorMessage(QString("Couldn't write header to file: ") + avErrorToQString(ret)); + return false; + } + + return true; +} +void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) +{ + static bool isFirstFrame = true; + if (isFirstFrame) { + timestamp = 0; + isFirstFrame = false; + } + + if (!mVideoWorker->isRunning()) { + qDebug() << "Encoder worker thread not running. Queuing frame."; + mPendingFrames.enqueue({pImage, timestamp}); + } + + else { + // First send any queued frames, then the latest one + while (!mPendingFrames.isEmpty()) { + AVFrame* avFrame = convertFrame(mPendingFrames.dequeue()); + if (avFrame) + mVideoWorker->queueFrame(avFrame); + } + + // note: if converting the frame turns out to be too slow to do here, it + // can always be done from the worker thread (in thta case, + // the worker's queue would contain ImageFrames rather than AVFrames) + + AVFrame* avFrame = convertFrame({pImage, timestamp}); + if (avFrame) + mVideoWorker->queueFrame(avFrame); + + // signal the worker that frames are available + mVideoWorker->mWaitCondition.wakeAll(); + } +} + +/** Convert a frame consisting of a QImage and timestamp to an AVFrame + * with the right pixel format and PTS + */ +AVFrame* UBFFmpegVideoEncoder::convertFrame(ImageFrame frame) +{ + AVFrame* avFrame = av_frame_alloc(); + + avFrame->format = mVideoStream->codec->pix_fmt; + avFrame->width = mVideoStream->codec->width; + avFrame->height = mVideoStream->codec->height; + avFrame->pts = mTimebase * frame.timestamp / 1000; + + const uchar * rgbImage = frame.image.bits(); + + const int in_linesize[1] = { frame.image.bytesPerLine() }; + + // Allocate the output image + if (av_image_alloc(avFrame->data, avFrame->linesize, mVideoStream->codec->width, + mVideoStream->codec->height, mVideoStream->codec->pix_fmt, 32) < 0) + { + setLastErrorMessage("Couldn't allocate image"); + return NULL; + } + + sws_scale(mSwsContext, + (const uint8_t* const*)&rgbImage, + in_linesize, + 0, + mVideoStream->codec->height, + avFrame->data, + avFrame->linesize); + + return avFrame; +} + +void UBFFmpegVideoEncoder::finishEncoding() +{ + qDebug() << "VideoEncoder::finishEncoding called"; + + // Some frames may not be encoded, so we call avcodec_encode_video2 until they're all done + + int gotOutput; + do { + // TODO: get rid of duplicated code (videoWorker does almost exactly this during encoding) + + AVPacket* packet = mVideoWorker->mPacket; + + if (avcodec_encode_video2(mVideoStream->codec, packet, NULL, &gotOutput) < 0) { + setLastErrorMessage("Couldn't encode frame to video"); + continue; + } + if (gotOutput) { + AVRational codecTimebase = mVideoStream->codec->time_base; + AVRational streamTimebase = mVideoStream->time_base; + + av_packet_rescale_ts(packet, codecTimebase, streamTimebase); + packet->stream_index = mVideoStream->index; + + av_interleaved_write_frame(mOutputFormatContext, packet); + av_packet_unref(packet); + } + } while (gotOutput); + + av_write_trailer(mOutputFormatContext); + + avio_close(mOutputFormatContext->pb); + avcodec_close(mVideoStream->codec); + sws_freeContext(mSwsContext); + avformat_free_context(mOutputFormatContext); + + emit encodingFinished(true); +} + +//------------------------------------------------------------------------- +// Worker +//------------------------------------------------------------------------- + +UBFFmpegVideoEncoderWorker::UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller) + : mController(controller) +{ + mStopRequested = false; + mIsRunning = false; + mPacket = new AVPacket(); +} + +UBFFmpegVideoEncoderWorker::~UBFFmpegVideoEncoderWorker() +{} + +void UBFFmpegVideoEncoderWorker::stopEncoding() +{ + qDebug() << "Video worker: stop requested"; + mStopRequested = true; + mWaitCondition.wakeAll(); +} + +void UBFFmpegVideoEncoderWorker::queueFrame(AVFrame* frame) +{ + mFrameQueueMutex.lock(); + mFrameQueue.enqueue(frame); + mFrameQueueMutex.unlock(); +} + +/** + * The main encoding function. Takes the queued image frames and + * assembles them into the video + */ +void UBFFmpegVideoEncoderWorker::runEncoding() +{ + mIsRunning = true; + + while (!mStopRequested) { + mFrameQueueMutex.lock(); + mWaitCondition.wait(&mFrameQueueMutex); + + while (!mFrameQueue.isEmpty()) { + writeLatestVideoFrame(); + } + + /* + while (!mAudioQueue.isEmpty()) { + writeLatestAudioFrame(); + } + */ + + mFrameQueueMutex.unlock(); + } + + emit encodingFinished(); +} + +void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame() +{ + AVFrame* frame = mFrameQueue.dequeue(); + + int gotOutput; + av_init_packet(mPacket); + mPacket->data = NULL; + mPacket->size = 0; + + // qDebug() << "Encoding frame to video. Pts: " << frame->pts << "/" << mController->mTimebase; + + if (avcodec_encode_video2(mController->mVideoStream->codec, mPacket, frame, &gotOutput) < 0) + emit error("Error encoding frame to video"); + + if (gotOutput) { + AVRational codecTimebase = mController->mVideoStream->codec->time_base; + AVRational streamTimebase = mController->mVideoStream->time_base; + + + // recalculate the timestamp to match the stream's timebase + av_packet_rescale_ts(mPacket, codecTimebase, streamTimebase); + mPacket->stream_index = mController->mVideoStream->index; + + // qDebug() << "Writing encoded packet to file; pts: " << mPacket->pts << "/" << streamTimebase.den; + + av_interleaved_write_frame(mController->mOutputFormatContext, mPacket); + av_packet_unref(mPacket); + } + + // Duct-tape solution. I assume there's a better way of doing this, but: + // some players like VLC show a black screen until the second frame (which + // can be several seconds after the first one). Simply duplicating the first frame + // seems to solve this problem, and also allows the thumbnail to be generated. + + static bool firstRun = true; + if (firstRun) { + firstRun = false; + frame->pts += 1; + mFrameQueue.enqueue(frame); // only works when the queue is empty at this point. todo: clean this up! + } + else + // free the frame + av_frame_free(&frame); +} + diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h new file mode 100644 index 00000000..4dacaaf6 --- /dev/null +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h @@ -0,0 +1,135 @@ +#ifndef UBFFMPEGVIDEOENCODER_H +#define UBFFMPEGVIDEOENCODER_H + +extern "C" { + #include + #include + #include + #include + #include + #include + #include + #include +} + +#include +#include + +#include +#include + +#include "podcast/UBAbstractVideoEncoder.h" + +class UBFFmpegVideoEncoderWorker; +class UBPodcastController; + +class UBFFmpegVideoEncoder : public UBAbstractVideoEncoder +{ + Q_OBJECT + + friend class UBFFmpegVideoEncoderWorker; + +public: + + UBFFmpegVideoEncoder(QObject* parent = NULL); + virtual ~UBFFmpegVideoEncoder(); + + bool start(); + bool stop(); + + void newPixmap(const QImage& pImage, long timestamp); + + QString videoFileExtension() const { return "mp4"; } + + QString lastErrorMessage() { return mLastErrorMessage; } + + void setRecordAudio(bool pRecordAudio) { mShouldRecordAudio = pRecordAudio; } + + +signals: + + void encodingFinished(bool ok); + +private slots: + + void setLastErrorMessage(const QString& pMessage); + void finishEncoding(); + +private: + + struct ImageFrame + { + QImage image; + long timestamp; // unit: ms + }; + + AVFrame* convertFrame(ImageFrame frame); + bool init(); + + // Queue for any pixmap that might be sent before the encoder is ready + QQueue mPendingFrames; + + QString mLastErrorMessage; + bool mShouldRecordAudio; + + QThread* mVideoEncoderThread; + UBFFmpegVideoEncoderWorker* mVideoWorker; + + // Muxer + AVFormatContext* mOutputFormatContext; + int mTimebase; + + // Video + AVStream* mVideoStream; + struct SwsContext * mSwsContext; + + // Audio + AVStream* mAudioStream; + + + FILE * mFile; + +}; + + +class UBFFmpegVideoEncoderWorker : public QObject +{ + Q_OBJECT + + friend class UBFFmpegVideoEncoder; + +public: + UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller); + ~UBFFmpegVideoEncoderWorker(); + + bool isRunning() { return mIsRunning; } + + void queueFrame(AVFrame* frame); + +public slots: + void runEncoding(); + void stopEncoding(); + +signals: + void encodingFinished(); + void error(QString message); + + +private: + void writeLatestVideoFrame(); + + UBFFmpegVideoEncoder* mController; + + // std::atomic is C++11. This won't work with msvc2010, so a + // newer compiler must be used if this is to be used on Windows + std::atomic mStopRequested; + std::atomic mIsRunning; + + QQueue mFrameQueue; + QMutex mFrameQueueMutex; + QWaitCondition mWaitCondition; + + AVPacket* mPacket; +}; + +#endif // UBFFMPEGVIDEOENCODER_H diff --git a/src/podcast/podcast.pri b/src/podcast/podcast.pri index af441814..0df7b13e 100644 --- a/src/podcast/podcast.pri +++ b/src/podcast/podcast.pri @@ -33,3 +33,28 @@ macx { OBJECTIVE_SOURCES += src/podcast/quicktime/UBQuickTimeFile.mm } + +linux-g++* { + HEADERS += src/podcast/ffmpeg/UBFFmpegVideoEncoder.h + + SOURCES += src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp + + + FFMPEG = /opt/ffmpeg + + INCLUDEPATH += $${FFMPEG}/include + DEPENDPATH += /usr/lib/x86_64-linux-gnu + + LIBS += -L $${FFMPEG}/lib -lavformat \ + -L $${FFMPEG}/lib -lavcodec \ + -L $${FFMPEG}/lib -lswscale \ + -L $${FFMPEG}/lib -lavutil \ + -lva-x11 \ + -lva \ + -lxcb-shm \ + -lxcb-xfixes \ + -lxcb-render -lxcb-shape -lxcb -lX11 -lasound -lSDL -lx264 -lpthread -lvpx -lvorbisenc -lvorbis -ltheoraenc -ltheoradec -logg -lopus -lmp3lame -lfreetype -lfdk-aac -lass -llzma -lbz2 -lz -ldl -lswresample -lswscale -lavutil -lm + + + QMAKE_CXXFLAGS += -std=c++11 # move this to OpenBoard.pro when we can use C++11 on all platforms +} From 11c207d7ee66e110606c28adcdd4adbe733380de Mon Sep 17 00:00:00 2001 From: Craig Watson Date: Wed, 4 May 2016 11:48:26 +0200 Subject: [PATCH 2/5] Podcasts on Linux: added audio support --- src/podcast/UBPodcastController.cpp | 3 + src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp | 346 +++++++++++++++++--- src/podcast/ffmpeg/UBFFmpegVideoEncoder.h | 48 ++- src/podcast/ffmpeg/UBMicrophoneInput.cpp | 211 ++++++++++++ src/podcast/ffmpeg/UBMicrophoneInput.h | 57 ++++ src/podcast/podcast.pri | 6 +- 6 files changed, 608 insertions(+), 63 deletions(-) create mode 100644 src/podcast/ffmpeg/UBMicrophoneInput.cpp create mode 100644 src/podcast/ffmpeg/UBMicrophoneInput.h diff --git a/src/podcast/UBPodcastController.cpp b/src/podcast/UBPodcastController.cpp index 20a96300..efe22027 100644 --- a/src/podcast/UBPodcastController.cpp +++ b/src/podcast/UBPodcastController.cpp @@ -66,6 +66,7 @@ #include "quicktime/UBAudioQueueRecorder.h" #elif defined(Q_OS_LINUX) #include "ffmpeg/UBFFmpegVideoEncoder.h" + #include "ffmpeg/UBMicrophoneInput.h" #endif #include "core/memcheck.h" @@ -808,6 +809,8 @@ QStringList UBPodcastController::audioRecordingDevices() devices = UBWaveRecorder::waveInDevices(); #elif defined(Q_OS_OSX) devices = UBAudioQueueRecorder::waveInDevices(); +#elif defined(Q_OS_LINUX) + devices = UBMicrophoneInput::availableDevicesNames(); #endif return devices; diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp index d2dd361a..09fcb5b4 100644 --- a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp @@ -20,18 +20,32 @@ QString avErrorToQString(int errnum) * * This class provides an interface between the screencast controller and the ffmpeg * back-end. It initializes the audio and video encoders and frees them when done; - * worker threads handle the actual encoding of frames. + * a worker thread handles the actual encoding and writing of frames. * */ UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent) : UBAbstractVideoEncoder(parent) , mOutputFormatContext(NULL) , mSwsContext(NULL) - , mFile(NULL) + , mShouldRecordAudio(true) + , mAudioInput(NULL) + , mSwrContext(NULL) + , mAudioOutBuffer(NULL) + , mAudioSampleRate(44100) + , mAudioFrameCount(0) { + if (mShouldRecordAudio) { + mAudioInput = new UBMicrophoneInput(); - mTimebase = 100 * framesPerSecond(); - qDebug() << "timebase: " << mTimebase; + connect(mAudioInput, SIGNAL(audioLevelChanged(quint8)), + this, SIGNAL(audioLevelChanged(quint8))); + + connect(mAudioInput, SIGNAL(dataAvailable(QByteArray)), + this, SLOT(onAudioAvailable(QByteArray))); + } + + mVideoTimebase = 100 * framesPerSecond(); + qDebug() << "timebase: " << mVideoTimebase; mVideoEncoderThread = new QThread; mVideoWorker = new UBFFmpegVideoEncoderWorker(this); @@ -58,6 +72,8 @@ UBFFmpegVideoEncoder::~UBFFmpegVideoEncoder() if (mVideoEncoderThread) delete mVideoEncoderThread; + if (mAudioInput) + delete mAudioInput; } void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage) @@ -66,12 +82,16 @@ void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage) mLastErrorMessage = pMessage; } + bool UBFFmpegVideoEncoder::start() { bool initialized = init(); - if (initialized) + if (initialized) { mVideoEncoderThread->start(); + if (mShouldRecordAudio) + mAudioInput->start(); + } return initialized; } @@ -82,12 +102,14 @@ bool UBFFmpegVideoEncoder::stop() mVideoWorker->stopEncoding(); + if (mShouldRecordAudio) + mAudioInput->stop(); + return true; } bool UBFFmpegVideoEncoder::init() { - // Initialize ffmpeg lib av_register_all(); avcodec_register_all(); @@ -96,7 +118,6 @@ bool UBFFmpegVideoEncoder::init() // Output format and context // -------------------------------------- - if (avformat_alloc_output_context2(&mOutputFormatContext, NULL, "mp4", NULL) < 0) { @@ -109,6 +130,7 @@ bool UBFFmpegVideoEncoder::init() // Video codec and context // ------------------------------------- + mVideoStream = avformat_new_stream(mOutputFormatContext, 0); AVCodec * videoCodec = avcodec_find_encoder(mOutputFormatContext->oformat->video_codec); if (!videoCodec) { @@ -116,16 +138,12 @@ bool UBFFmpegVideoEncoder::init() return false; } - mVideoStream = avformat_new_stream(mOutputFormatContext, 0); - mVideoStream->time_base = {1, mTimebase}; - - avcodec_get_context_defaults3(mVideoStream->codec, videoCodec); AVCodecContext* c = avcodec_alloc_context3(videoCodec); c->bit_rate = videoBitsPerSecond(); c->width = videoSize().width(); c->height = videoSize().height(); - c->time_base = {1, mTimebase}; + c->time_base = {1, mVideoTimebase}; c->gop_size = 10; c->max_b_frames = 0; c->pix_fmt = AV_PIX_FMT_YUV420P; @@ -161,10 +179,77 @@ bool UBFFmpegVideoEncoder::init() // Audio codec and context // ------------------------------------- - /* - AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec); - mAudioStream = avformat_new_stream(mOutputFormatContext, audioCodec); - */ + if (mShouldRecordAudio) { + + // Microphone input + if (!mAudioInput->init()) { + setLastErrorMessage("Couldn't initialize audio input"); + return false; + } + + + int inChannelCount = mAudioInput->channelCount(); + int inSampleRate = mAudioInput->sampleRate(); + int inSampleSize = mAudioInput->sampleSize(); + + qDebug() << "inChannelCount = " << inChannelCount; + qDebug() << "inSampleRate = " << inSampleRate; + qDebug() << "inSampleSize = " << inSampleSize; + + // Codec + AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec); + + if (!audioCodec) { + setLastErrorMessage("Audio codec not found"); + return false; + } + + mAudioStream = avformat_new_stream(mOutputFormatContext, audioCodec); + mAudioStream->id = mOutputFormatContext->nb_streams-1; + + c = mAudioStream->codec; + + c->bit_rate = 96000; + c->sample_fmt = audioCodec->sample_fmts[0]; // FLTP by default for AAC + c->sample_rate = mAudioSampleRate; + c->channels = 2; + c->channel_layout = av_get_default_channel_layout(c->channels); + c->profile = FF_PROFILE_AAC_MAIN; + c->time_base = {1, mAudioSampleRate}; + + if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER) + c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + + ret = avcodec_open2(c, audioCodec, NULL); + + if (ret < 0) { + setLastErrorMessage(QString("Couldn't open audio codec: ") + avErrorToQString(ret)); + return false; + } + + // Resampling / format converting context + mSwrContext = swr_alloc(); + if (!mSwrContext) { + setLastErrorMessage("Could not allocate resampler context"); + return false; + } + + av_opt_set_int(mSwrContext, "in_channel_count", inChannelCount, 0); + av_opt_set_int(mSwrContext, "in_sample_rate", inSampleRate, 0); + av_opt_set_sample_fmt(mSwrContext, "in_sample_fmt", (AVSampleFormat)mAudioInput->sampleFormat(), 0); + av_opt_set_int(mSwrContext, "out_channel_count", c->channels, 0); + av_opt_set_int(mSwrContext, "out_sample_rate", c->sample_rate, 0); + av_opt_set_sample_fmt(mSwrContext, "out_sample_fmt", c->sample_fmt, 0); + + ret = swr_init(mSwrContext); + if (ret < 0) { + setLastErrorMessage(QString("Couldn't initialize the resampling context: ") + avErrorToQString(ret)); + return false; + } + + // Buffer for resampled/converted audio + mAudioOutBuffer = av_audio_fifo_alloc(c->sample_fmt, c->channels, c->frame_size); + } // Open the output file @@ -185,8 +270,14 @@ bool UBFFmpegVideoEncoder::init() return true; } + +/** + * This function should be called every time a new "screenshot" is ready. + * The image is converted to the right format and sent to the encoder. + */ void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) { + // really necessary? static bool isFirstFrame = true; if (isFirstFrame) { timestamp = 0; @@ -201,16 +292,16 @@ void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) else { // First send any queued frames, then the latest one while (!mPendingFrames.isEmpty()) { - AVFrame* avFrame = convertFrame(mPendingFrames.dequeue()); + AVFrame* avFrame = convertImageFrame(mPendingFrames.dequeue()); if (avFrame) mVideoWorker->queueFrame(avFrame); } // note: if converting the frame turns out to be too slow to do here, it - // can always be done from the worker thread (in thta case, + // can always be done from the worker thread (in that case, // the worker's queue would contain ImageFrames rather than AVFrames) - AVFrame* avFrame = convertFrame({pImage, timestamp}); + AVFrame* avFrame = convertImageFrame({pImage, timestamp}); if (avFrame) mVideoWorker->queueFrame(avFrame); @@ -219,17 +310,18 @@ void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) } } -/** Convert a frame consisting of a QImage and timestamp to an AVFrame +/** + * Convert a frame consisting of a QImage and timestamp to an AVFrame * with the right pixel format and PTS */ -AVFrame* UBFFmpegVideoEncoder::convertFrame(ImageFrame frame) +AVFrame* UBFFmpegVideoEncoder::convertImageFrame(ImageFrame frame) { AVFrame* avFrame = av_frame_alloc(); avFrame->format = mVideoStream->codec->pix_fmt; avFrame->width = mVideoStream->codec->width; avFrame->height = mVideoStream->codec->height; - avFrame->pts = mTimebase * frame.timestamp / 1000; + avFrame->pts = mVideoTimebase * frame.timestamp / 1000; const uchar * rgbImage = frame.image.bits(); @@ -254,6 +346,93 @@ AVFrame* UBFFmpegVideoEncoder::convertFrame(ImageFrame frame) return avFrame; } +void UBFFmpegVideoEncoder::onAudioAvailable(QByteArray data) +{ + if (!data.isEmpty()) + processAudio(data); +} + +/** +* Resample and convert audio to match the encoder's settings and queue the +* output. If enough output data is available, it is packaged into AVFrames and +* sent to the encoder thread. +*/ +void UBFFmpegVideoEncoder::processAudio(QByteArray &data) +{ + int ret; + AVCodecContext* codecContext = mAudioStream->codec; + + const char * inSamples = data.constData(); + + // The number of samples (per channel) in the input + int inSamplesCount = data.size() / ((mAudioInput->sampleSize() / 8) * mAudioInput->channelCount()); + + // The number of samples we will get after conversion + int outSamplesCount = swr_get_out_samples(mSwrContext, inSamplesCount); + + // Allocate output samples + uint8_t ** outSamples = NULL; + int outSamplesLineSize; + + ret = av_samples_alloc_array_and_samples(&outSamples, &outSamplesLineSize, + codecContext->channels, outSamplesCount, + codecContext->sample_fmt, 0); + if (ret < 0) { + qDebug() << "Could not allocate audio samples" << avErrorToQString(ret); + return; + } + + // Convert to destination format + ret = swr_convert(mSwrContext, + outSamples, outSamplesCount, + (const uint8_t **)&inSamples, inSamplesCount); + if (ret < 0) { + qDebug() << "Error converting audio samples: " << avErrorToQString(ret); + return; + } + + // Append the converted samples to the out buffer. + ret = av_audio_fifo_write(mAudioOutBuffer, (void**)outSamples, outSamplesCount); + if (ret < 0) { + qDebug() << "Could not write to FIFO queue: " << avErrorToQString(ret); + return; + } + + // Keep the data queued until next call if the encoder thread isn't running + if (!mVideoWorker->isRunning()) + return; + + bool framesAdded = false; + while (av_audio_fifo_size(mAudioOutBuffer) > codecContext->frame_size) { + AVFrame * avFrame = av_frame_alloc(); + avFrame->nb_samples = codecContext->frame_size; + avFrame->channel_layout = codecContext->channel_layout; + avFrame->format = codecContext->sample_fmt; + avFrame->sample_rate = codecContext->sample_rate; + avFrame->pts = mAudioFrameCount; + + ret = av_frame_get_buffer(avFrame, 0); + if (ret < 0) { + qDebug() << "Couldn't allocate frame: " << avErrorToQString(ret); + break; + } + + ret = av_audio_fifo_read(mAudioOutBuffer, (void**)avFrame->data, codecContext->frame_size); + if (ret < 0) + qDebug() << "Could not read from FIFO queue: " << avErrorToQString(ret); + + else { + mAudioFrameCount += codecContext->frame_size; + + mVideoWorker->queueAudio(avFrame); + framesAdded = true; + } + } + + if (framesAdded) + mVideoWorker->mWaitCondition.wakeAll(); +} + void UBFFmpegVideoEncoder::finishEncoding() { qDebug() << "VideoEncoder::finishEncoding called"; @@ -264,7 +443,7 @@ void UBFFmpegVideoEncoder::finishEncoding() do { // TODO: get rid of duplicated code (videoWorker does almost exactly this during encoding) - AVPacket* packet = mVideoWorker->mPacket; + AVPacket* packet = mVideoWorker->mVideoPacket; if (avcodec_encode_video2(mVideoStream->codec, packet, NULL, &gotOutput) < 0) { setLastErrorMessage("Couldn't encode frame to video"); @@ -272,9 +451,9 @@ void UBFFmpegVideoEncoder::finishEncoding() } if (gotOutput) { AVRational codecTimebase = mVideoStream->codec->time_base; - AVRational streamTimebase = mVideoStream->time_base; + AVRational streamVideoTimebase = mVideoStream->time_base; - av_packet_rescale_ts(packet, codecTimebase, streamTimebase); + av_packet_rescale_ts(packet, codecTimebase, streamVideoTimebase); packet->stream_index = mVideoStream->index; av_interleaved_write_frame(mOutputFormatContext, packet); @@ -282,16 +461,48 @@ void UBFFmpegVideoEncoder::finishEncoding() } } while (gotOutput); + if (mShouldRecordAudio) { + + int gotOutput, ret; + do { + + AVPacket* packet = mVideoWorker->mAudioPacket; + + ret = avcodec_encode_audio2(mAudioStream->codec, packet, NULL, &gotOutput); + if (ret < 0) + setLastErrorMessage("Couldn't encode frame to audio"); + + else if (gotOutput) { + AVRational codecTimebase = mAudioStream->codec->time_base; + AVRational streamVideoTimebase = mAudioStream->time_base; + + av_packet_rescale_ts(packet, codecTimebase, streamVideoTimebase); + packet->stream_index = mAudioStream->index; + + av_interleaved_write_frame(mOutputFormatContext, packet); + av_packet_unref(packet); + } + } while (gotOutput); + + } + av_write_trailer(mOutputFormatContext); avio_close(mOutputFormatContext->pb); avcodec_close(mVideoStream->codec); sws_freeContext(mSwsContext); + + if (mShouldRecordAudio) { + avcodec_close(mAudioStream->codec); + swr_free(&mSwrContext); + } + avformat_free_context(mOutputFormatContext); emit encodingFinished(true); } + //------------------------------------------------------------------------- // Worker //------------------------------------------------------------------------- @@ -301,7 +512,8 @@ UBFFmpegVideoEncoderWorker::UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* con { mStopRequested = false; mIsRunning = false; - mPacket = new AVPacket(); + mVideoPacket = new AVPacket(); + mAudioPacket = new AVPacket(); } UBFFmpegVideoEncoderWorker::~UBFFmpegVideoEncoderWorker() @@ -316,11 +528,23 @@ void UBFFmpegVideoEncoderWorker::stopEncoding() void UBFFmpegVideoEncoderWorker::queueFrame(AVFrame* frame) { - mFrameQueueMutex.lock(); - mFrameQueue.enqueue(frame); - mFrameQueueMutex.unlock(); + if (frame) { + mFrameQueueMutex.lock(); + mImageQueue.enqueue(frame); + mFrameQueueMutex.unlock(); + } } +void UBFFmpegVideoEncoderWorker::queueAudio(AVFrame* frame) +{ + if (frame) { + mFrameQueueMutex.lock(); + mAudioQueue.enqueue(frame); + mFrameQueueMutex.unlock(); + } +} + + /** * The main encoding function. Takes the queued image frames and * assembles them into the video @@ -333,15 +557,13 @@ void UBFFmpegVideoEncoderWorker::runEncoding() mFrameQueueMutex.lock(); mWaitCondition.wait(&mFrameQueueMutex); - while (!mFrameQueue.isEmpty()) { + while (!mImageQueue.isEmpty()) { writeLatestVideoFrame(); } - /* while (!mAudioQueue.isEmpty()) { writeLatestAudioFrame(); } - */ mFrameQueueMutex.unlock(); } @@ -351,31 +573,31 @@ void UBFFmpegVideoEncoderWorker::runEncoding() void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame() { - AVFrame* frame = mFrameQueue.dequeue(); + AVFrame* frame = mImageQueue.dequeue(); int gotOutput; - av_init_packet(mPacket); - mPacket->data = NULL; - mPacket->size = 0; + av_init_packet(mVideoPacket); + mVideoPacket->data = NULL; + mVideoPacket->size = 0; - // qDebug() << "Encoding frame to video. Pts: " << frame->pts << "/" << mController->mTimebase; + // qDebug() << "Encoding frame to video. Pts: " << frame->pts << "/" << mController->mVideoTimebase; - if (avcodec_encode_video2(mController->mVideoStream->codec, mPacket, frame, &gotOutput) < 0) - emit error("Error encoding frame to video"); + if (avcodec_encode_video2(mController->mVideoStream->codec, mVideoPacket, frame, &gotOutput) < 0) + emit error("Error encoding video frame"); if (gotOutput) { AVRational codecTimebase = mController->mVideoStream->codec->time_base; - AVRational streamTimebase = mController->mVideoStream->time_base; + AVRational streamVideoTimebase = mController->mVideoStream->time_base; // recalculate the timestamp to match the stream's timebase - av_packet_rescale_ts(mPacket, codecTimebase, streamTimebase); - mPacket->stream_index = mController->mVideoStream->index; + av_packet_rescale_ts(mVideoPacket, codecTimebase, streamVideoTimebase); + mVideoPacket->stream_index = mController->mVideoStream->index; - // qDebug() << "Writing encoded packet to file; pts: " << mPacket->pts << "/" << streamTimebase.den; + // qDebug() << "Writing encoded packet to file; pts: " << mVideoPacket->pts << "/" << streamVideoTimebase.den; - av_interleaved_write_frame(mController->mOutputFormatContext, mPacket); - av_packet_unref(mPacket); + av_interleaved_write_frame(mController->mOutputFormatContext, mVideoPacket); + av_packet_unref(mVideoPacket); } // Duct-tape solution. I assume there's a better way of doing this, but: @@ -387,10 +609,42 @@ void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame() if (firstRun) { firstRun = false; frame->pts += 1; - mFrameQueue.enqueue(frame); // only works when the queue is empty at this point. todo: clean this up! + mImageQueue.enqueue(frame); // only works when the queue is empty at this point. todo: clean this up! } else // free the frame av_frame_free(&frame); } +void UBFFmpegVideoEncoderWorker::writeLatestAudioFrame() +{ + AVFrame *frame = mAudioQueue.dequeue(); + + int gotOutput, ret; + + av_init_packet(mAudioPacket); + mAudioPacket->data = NULL; + mAudioPacket->size = 0; + + //qDebug() << "Encoding audio frame"; + + ret = avcodec_encode_audio2(mController->mAudioStream->codec, mAudioPacket, frame, &gotOutput); + if (ret < 0) + emit error(QString("Error encoding audio frame: ") + avErrorToQString(ret)); + + else if (gotOutput) { + //qDebug() << "Writing audio frame to stream"; + + AVRational codecTimebase = mController->mAudioStream->codec->time_base; + AVRational streamVideoTimebase = mController->mAudioStream->time_base; + + av_packet_rescale_ts(mAudioPacket, codecTimebase, streamVideoTimebase); + mAudioPacket->stream_index = mController->mAudioStream->index; + + av_interleaved_write_frame(mController->mOutputFormatContext, mAudioPacket); + av_packet_unref(mAudioPacket); + } + + av_frame_free(&frame); +} + diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h index 4dacaaf6..83f74fd7 100644 --- a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h @@ -5,20 +5,23 @@ extern "C" { #include #include #include + #include #include #include #include #include + #include #include + #include } #include -#include #include #include #include "podcast/UBAbstractVideoEncoder.h" +#include "podcast/ffmpeg/UBMicrophoneInput.h" class UBFFmpegVideoEncoderWorker; class UBPodcastController; @@ -45,7 +48,6 @@ public: void setRecordAudio(bool pRecordAudio) { mShouldRecordAudio = pRecordAudio; } - signals: void encodingFinished(bool ok); @@ -53,6 +55,7 @@ signals: private slots: void setLastErrorMessage(const QString& pMessage); + void onAudioAvailable(QByteArray data); void finishEncoding(); private: @@ -63,32 +66,42 @@ private: long timestamp; // unit: ms }; - AVFrame* convertFrame(ImageFrame frame); + AVFrame* convertImageFrame(ImageFrame frame); + AVFrame* convertAudio(QByteArray data); + void processAudio(QByteArray& data); bool init(); - // Queue for any pixmap that might be sent before the encoder is ready - QQueue mPendingFrames; - QString mLastErrorMessage; - bool mShouldRecordAudio; QThread* mVideoEncoderThread; UBFFmpegVideoEncoderWorker* mVideoWorker; // Muxer + // ------------------------------------------ AVFormatContext* mOutputFormatContext; - int mTimebase; + AVStream* mVideoStream; + AVStream* mAudioStream; // Video - AVStream* mVideoStream; + // ------------------------------------------ + QQueue mPendingFrames; struct SwsContext * mSwsContext; - // Audio - AVStream* mAudioStream; + int mVideoTimebase; + // Audio + // ------------------------------------------ + bool mShouldRecordAudio; - FILE * mFile; + UBMicrophoneInput * mAudioInput; + struct SwrContext * mSwrContext; + /// Queue for audio that has been rescaled/converted but not encoded yet + AVAudioFifo *mAudioOutBuffer; + /// Sample rate for encoded audio + int mAudioSampleRate; + /// Total audio frames sent to encoder + int mAudioFrameCount; }; @@ -105,6 +118,7 @@ public: bool isRunning() { return mIsRunning; } void queueFrame(AVFrame* frame); + void queueAudio(AVFrame *frame); public slots: void runEncoding(); @@ -117,19 +131,23 @@ signals: private: void writeLatestVideoFrame(); + void writeLatestAudioFrame(); UBFFmpegVideoEncoder* mController; // std::atomic is C++11. This won't work with msvc2010, so a - // newer compiler must be used if this is to be used on Windows + // newer compiler must be used if this class is to be used on Windows std::atomic mStopRequested; std::atomic mIsRunning; - QQueue mFrameQueue; + QQueue mImageQueue; + QQueue mAudioQueue; + QMutex mFrameQueueMutex; QWaitCondition mWaitCondition; - AVPacket* mPacket; + AVPacket* mVideoPacket; + AVPacket* mAudioPacket; }; #endif // UBFFMPEGVIDEOENCODER_H diff --git a/src/podcast/ffmpeg/UBMicrophoneInput.cpp b/src/podcast/ffmpeg/UBMicrophoneInput.cpp new file mode 100644 index 00000000..b9c3cf2c --- /dev/null +++ b/src/podcast/ffmpeg/UBMicrophoneInput.cpp @@ -0,0 +1,211 @@ +#include "UBMicrophoneInput.h" + +UBMicrophoneInput::UBMicrophoneInput() + : mAudioInput(NULL) + , mIODevice(NULL) + , mSeekPos(0) +{ +} + +UBMicrophoneInput::~UBMicrophoneInput() +{ + if (mAudioInput) + delete mAudioInput; +} + +int UBMicrophoneInput::channelCount() +{ + return mAudioFormat.channelCount(); +} + +int UBMicrophoneInput::sampleRate() +{ + return mAudioFormat.sampleRate(); +} + +/* Return the sample size in bits */ +int UBMicrophoneInput::sampleSize() +{ + return mAudioFormat.sampleSize(); +} + +/** Return the sample format in FFMpeg style (AVSampleFormat enum) */ +int UBMicrophoneInput::sampleFormat() +{ + enum AVSampleFormat { + AV_SAMPLE_FMT_NONE = -1, + AV_SAMPLE_FMT_U8, + AV_SAMPLE_FMT_S16, + AV_SAMPLE_FMT_S32, + AV_SAMPLE_FMT_FLT, + AV_SAMPLE_FMT_DBL, + AV_SAMPLE_FMT_U8P, + AV_SAMPLE_FMT_S16P, + AV_SAMPLE_FMT_S32P, + AV_SAMPLE_FMT_FLTP, + AV_SAMPLE_FMT_DBLP, + AV_SAMPLE_FMT_NB + }; + + int sampleSize = mAudioFormat.sampleSize(); + QAudioFormat::SampleType sampleType = mAudioFormat.sampleType(); + + // qDebug() << "Input sample format: " << sampleSize << "bits " << sampleType; + + switch (sampleType) { + case QAudioFormat::Unknown: + return AV_SAMPLE_FMT_NONE; + + case QAudioFormat::SignedInt: + if (sampleSize == 16) + return AV_SAMPLE_FMT_S16; + if (sampleSize == 32) + return AV_SAMPLE_FMT_S32; + + case QAudioFormat::UnSignedInt: + if (sampleSize == 8) + return AV_SAMPLE_FMT_U8; + + case QAudioFormat::Float: + return AV_SAMPLE_FMT_FLT; + + default: + return AV_SAMPLE_FMT_NONE; + } +} + +QString UBMicrophoneInput::codec() +{ + return mAudioFormat.codec(); +} + +qint64 UBMicrophoneInput::processUSecs() const +{ + return mAudioInput->processedUSecs(); +} + +bool UBMicrophoneInput::init() +{ + if (mAudioDeviceInfo.isNull()) { + qWarning("No audio input device selected; using default"); + mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice(); + } + + qDebug() << "Input device name: " << mAudioDeviceInfo.deviceName(); + + mAudioFormat = mAudioDeviceInfo.preferredFormat(); + + mAudioInput = new QAudioInput(mAudioDeviceInfo, mAudioFormat, NULL); + //mAudioInput->setNotifyInterval(100); + + connect(mAudioInput, SIGNAL(stateChanged(QAudio::State)), + this, SLOT(onAudioInputStateChanged(QAudio::State))); + + return true; +} + +void UBMicrophoneInput::start() +{ + qDebug() << "starting audio input"; + + mIODevice = mAudioInput->start(); + + connect(mIODevice, SIGNAL(readyRead()), + this, SLOT(onDataReady())); + + if (mAudioInput->error() == QAudio::OpenError) + qWarning() << "Error opening audio input"; +} + +void UBMicrophoneInput::stop() +{ + mAudioInput->stop(); +} + +QStringList UBMicrophoneInput::availableDevicesNames() +{ + QStringList names; + QList devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput); + + foreach (QAudioDeviceInfo device, devices) { + names.push_back(device.deviceName()); + } + + return names; +} + +void UBMicrophoneInput::setInputDevice(QString name) +{ + if (name.isEmpty()) { + mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice(); + return; + } + + QList devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput); + bool found = false; + + foreach (QAudioDeviceInfo device, devices) { + if (device.deviceName() == name) { + mAudioDeviceInfo = device; + found = true; + break; + } + } + + if (!found) { + qWarning() << "Audio input device not found; using default instead"; + mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice(); + } + +} + +void UBMicrophoneInput::onDataReady() +{ + int numBytes = mAudioInput->bytesReady(); + + if (numBytes > 0) + emit dataAvailable(mIODevice->read(numBytes)); +} + +void UBMicrophoneInput::onAudioInputStateChanged(QAudio::State state) +{ + qDebug() << "Audio input state changed to " << state; + switch (state) { + case QAudio::StoppedState: + if (mAudioInput->error() != QAudio::NoError) { + emit error(getErrorString(mAudioInput->error())); + } + break; + + // handle other states? + + default: + break; + } +} + + +/** + * @brief Return a meaningful error string based on QAudio error codes + */ +QString UBMicrophoneInput::getErrorString(QAudio::Error errorCode) +{ + switch (errorCode) { + case QAudio::NoError : + return ""; + + case QAudio::OpenError : + return "Couldn't open the audio device"; + + case QAudio::IOError : + return "Error reading from audio device"; + + case QAudio::UnderrunError : + return "Underrun error"; + + case QAudio::FatalError : + return "Fatal error; audio device unusable"; + + } + return ""; +} diff --git a/src/podcast/ffmpeg/UBMicrophoneInput.h b/src/podcast/ffmpeg/UBMicrophoneInput.h new file mode 100644 index 00000000..8ba306d0 --- /dev/null +++ b/src/podcast/ffmpeg/UBMicrophoneInput.h @@ -0,0 +1,57 @@ +#ifndef UBMICROPHONEINPUT_H +#define UBMICROPHONEINPUT_H + +#include +#include + +/** + * @brief The UBMicrophoneInput class captures uncompressed sound from a microphone + */ +class UBMicrophoneInput : public QObject +{ + Q_OBJECT + +public: + UBMicrophoneInput(); + virtual ~UBMicrophoneInput(); + + bool init(); + void start(); + void stop(); + + static QStringList availableDevicesNames(); + void setInputDevice(QString name = ""); + + int channelCount(); + int sampleRate(); + int sampleSize(); + int sampleFormat(); + QString codec(); + + qint64 processUSecs() const; + +signals: + /// Send the new volume, between 0 and 255 + void audioLevelChanged(quint8 level); + + /// Emitted when new audio data is available + void dataAvailable(QByteArray data); + + void error(QString message); + +private slots: + void onAudioInputStateChanged(QAudio::State state); + void onDataReady(); + +private: + QString getErrorString(QAudio::Error errorCode); + + QAudioInput* mAudioInput; + QIODevice * mIODevice; + QAudioDeviceInfo mAudioDeviceInfo; + QAudioFormat mAudioFormat; + + qint64 mSeekPos; +}; + +#endif // UBMICROPHONEINPUT_H diff --git a/src/podcast/podcast.pri b/src/podcast/podcast.pri index 0df7b13e..5b10ee9b 100644 --- a/src/podcast/podcast.pri +++ b/src/podcast/podcast.pri @@ -3,13 +3,15 @@ HEADERS += src/podcast/UBPodcastController.h \ src/podcast/UBAbstractVideoEncoder.h \ src/podcast/UBPodcastRecordingPalette.h \ src/podcast/youtube/UBYouTubePublisher.h \ - src/podcast/intranet/UBIntranetPodcastPublisher.h + src/podcast/intranet/UBIntranetPodcastPublisher.h \ + $$PWD/ffmpeg/UBMicrophoneInput.h SOURCES += src/podcast/UBPodcastController.cpp \ src/podcast/UBAbstractVideoEncoder.cpp \ src/podcast/UBPodcastRecordingPalette.cpp \ src/podcast/youtube/UBYouTubePublisher.cpp \ - src/podcast/intranet/UBIntranetPodcastPublisher.cpp + src/podcast/intranet/UBIntranetPodcastPublisher.cpp \ + $$PWD/ffmpeg/UBMicrophoneInput.cpp win32 { From 518b7d26d44c3c8e914609eee70a650e533a7500 Mon Sep 17 00:00:00 2001 From: Craig Watson Date: Fri, 6 May 2016 12:59:42 +0200 Subject: [PATCH 3/5] Clean-up and removal of duplicate code --- src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp | 243 +++++++------------- src/podcast/ffmpeg/UBFFmpegVideoEncoder.h | 15 +- 2 files changed, 96 insertions(+), 162 deletions(-) diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp index 09fcb5b4..498fbae4 100644 --- a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp @@ -1,10 +1,8 @@ #include "UBFFmpegVideoEncoder.h" -// Future proofing -#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1) -#define av_frame_alloc avcodec_alloc_frame -#define av_frame_free avcodec_free_frame -#endif +//------------------------------------------------------------------------- +// Utility functions +//------------------------------------------------------------------------- QString avErrorToQString(int errnum) { @@ -15,14 +13,51 @@ QString avErrorToQString(int errnum) } /** - * @brief Constructor for the ffmpeg video encoder - * - * - * This class provides an interface between the screencast controller and the ffmpeg - * back-end. It initializes the audio and video encoders and frees them when done; - * a worker thread handles the actual encoding and writing of frames. + * @brief Write a given frame to the audio stream or, if a null frame is passed, flush the stream. * + * @param frame An AVFrame to be written to the stream, or NULL to flush the stream + * @param packet A (reusable) packet, used to temporarily store frame data + * @param stream The stream to write to + * @param outputFormatContext The output format context */ +void writeFrame(AVFrame *frame, AVPacket *packet, AVStream *stream, AVFormatContext *outputFormatContext) +{ + int gotOutput, ret; + + av_init_packet(packet); + + do { + if (stream->codec->codec_type == AVMEDIA_TYPE_AUDIO) + ret = avcodec_encode_audio2(stream->codec, packet, frame, &gotOutput); + else + ret = avcodec_encode_video2(stream->codec, packet, frame, &gotOutput); + + if (ret < 0) + qWarning() << "Couldn't encode audio frame: " << avErrorToQString(ret); + + else if (gotOutput) { + AVRational codecTimebase = stream->codec->time_base; + AVRational streamVideoTimebase = stream->time_base; + + av_packet_rescale_ts(packet, codecTimebase, streamVideoTimebase); + packet->stream_index = stream->index; + + av_interleaved_write_frame(outputFormatContext, packet); + av_packet_unref(packet); + } + + } while (gotOutput && !frame); +} + +void flushStream(AVPacket *packet, AVStream *stream, AVFormatContext *outputFormatContext) +{ + writeFrame(NULL, packet, stream, outputFormatContext); +} + +//------------------------------------------------------------------------- +// UBFFmpegVideoEncoder +//------------------------------------------------------------------------- + UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent) : UBAbstractVideoEncoder(parent) , mOutputFormatContext(NULL) @@ -34,18 +69,8 @@ UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent) , mAudioSampleRate(44100) , mAudioFrameCount(0) { - if (mShouldRecordAudio) { - mAudioInput = new UBMicrophoneInput(); - - connect(mAudioInput, SIGNAL(audioLevelChanged(quint8)), - this, SIGNAL(audioLevelChanged(quint8))); - - connect(mAudioInput, SIGNAL(dataAvailable(QByteArray)), - this, SLOT(onAudioAvailable(QByteArray))); - } mVideoTimebase = 100 * framesPerSecond(); - qDebug() << "timebase: " << mVideoTimebase; mVideoEncoderThread = new QThread; mVideoWorker = new UBFFmpegVideoEncoderWorker(this); @@ -78,7 +103,7 @@ UBFFmpegVideoEncoder::~UBFFmpegVideoEncoder() void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage) { - qDebug() << "FFmpeg video encoder:" << pMessage; + qWarning() << "FFmpeg video encoder:" << pMessage; mLastErrorMessage = pMessage; } @@ -182,12 +207,20 @@ bool UBFFmpegVideoEncoder::init() if (mShouldRecordAudio) { // Microphone input + + mAudioInput = new UBMicrophoneInput(); + + connect(mAudioInput, SIGNAL(audioLevelChanged(quint8)), + this, SIGNAL(audioLevelChanged(quint8))); + + connect(mAudioInput, SIGNAL(dataAvailable(QByteArray)), + this, SLOT(onAudioAvailable(QByteArray))); + if (!mAudioInput->init()) { setLastErrorMessage("Couldn't initialize audio input"); return false; } - int inChannelCount = mAudioInput->channelCount(); int inSampleRate = mAudioInput->sampleRate(); int inSampleSize = mAudioInput->sampleSize(); @@ -197,6 +230,7 @@ bool UBFFmpegVideoEncoder::init() qDebug() << "inSampleSize = " << inSampleSize; // Codec + AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec); if (!audioCodec) { @@ -227,7 +261,8 @@ bool UBFFmpegVideoEncoder::init() return false; } - // Resampling / format converting context + // The input (raw sound from the microphone) may not match the codec's sampling rate, + // sample format or number of channels; we use libswresample to convert and resample it mSwrContext = swr_alloc(); if (!mSwrContext) { setLastErrorMessage("Could not allocate resampler context"); @@ -277,13 +312,6 @@ bool UBFFmpegVideoEncoder::init() */ void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) { - // really necessary? - static bool isFirstFrame = true; - if (isFirstFrame) { - timestamp = 0; - isFirstFrame = false; - } - if (!mVideoWorker->isRunning()) { qDebug() << "Encoder worker thread not running. Queuing frame."; mPendingFrames.enqueue({pImage, timestamp}); @@ -294,7 +322,7 @@ void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) while (!mPendingFrames.isEmpty()) { AVFrame* avFrame = convertImageFrame(mPendingFrames.dequeue()); if (avFrame) - mVideoWorker->queueFrame(avFrame); + mVideoWorker->queueVideoFrame(avFrame); } // note: if converting the frame turns out to be too slow to do here, it @@ -303,7 +331,7 @@ void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) AVFrame* avFrame = convertImageFrame({pImage, timestamp}); if (avFrame) - mVideoWorker->queueFrame(avFrame); + mVideoWorker->queueVideoFrame(avFrame); // signal the worker that frames are available mVideoWorker->mWaitCondition.wakeAll(); @@ -331,7 +359,7 @@ AVFrame* UBFFmpegVideoEncoder::convertImageFrame(ImageFrame frame) if (av_image_alloc(avFrame->data, avFrame->linesize, mVideoStream->codec->width, mVideoStream->codec->height, mVideoStream->codec->pix_fmt, 32) < 0) { - setLastErrorMessage("Couldn't allocate image"); + qWarning() << "Couldn't allocate image"; return NULL; } @@ -378,7 +406,7 @@ void UBFFmpegVideoEncoder::processAudio(QByteArray &data) codecContext->channels, outSamplesCount, codecContext->sample_fmt, 0); if (ret < 0) { - qDebug() << "Could not allocate audio samples" << avErrorToQString(ret); + qWarning() << "Could not allocate audio samples" << avErrorToQString(ret); return; } @@ -387,14 +415,14 @@ void UBFFmpegVideoEncoder::processAudio(QByteArray &data) outSamples, outSamplesCount, (const uint8_t **)&inSamples, inSamplesCount); if (ret < 0) { - qDebug() << "Error converting audio samples: " << avErrorToQString(ret); + qWarning() << "Error converting audio samples: " << avErrorToQString(ret); return; } // Append the converted samples to the out buffer. ret = av_audio_fifo_write(mAudioOutBuffer, (void**)outSamples, outSamplesCount); if (ret < 0) { - qDebug() << "Could not write to FIFO queue: " << avErrorToQString(ret); + qWarning() << "Could not write to FIFO queue: " << avErrorToQString(ret); return; } @@ -413,18 +441,18 @@ void UBFFmpegVideoEncoder::processAudio(QByteArray &data) ret = av_frame_get_buffer(avFrame, 0); if (ret < 0) { - qDebug() << "Couldn't allocate frame: " << avErrorToQString(ret); + qWarning() << "Couldn't allocate frame: " << avErrorToQString(ret); break; } ret = av_audio_fifo_read(mAudioOutBuffer, (void**)avFrame->data, codecContext->frame_size); if (ret < 0) - qDebug() << "Could not read from FIFO queue: " << avErrorToQString(ret); + qWarning() << "Could not read from FIFO queue: " << avErrorToQString(ret); else { mAudioFrameCount += codecContext->frame_size; - mVideoWorker->queueAudio(avFrame); + mVideoWorker->queueAudioFrame(avFrame); framesAdded = true; } } @@ -437,58 +465,14 @@ void UBFFmpegVideoEncoder::finishEncoding() { qDebug() << "VideoEncoder::finishEncoding called"; - // Some frames may not be encoded, so we call avcodec_encode_video2 until they're all done - - int gotOutput; - do { - // TODO: get rid of duplicated code (videoWorker does almost exactly this during encoding) + flushStream(mVideoWorker->mVideoPacket, mVideoStream, mOutputFormatContext); - AVPacket* packet = mVideoWorker->mVideoPacket; - - if (avcodec_encode_video2(mVideoStream->codec, packet, NULL, &gotOutput) < 0) { - setLastErrorMessage("Couldn't encode frame to video"); - continue; - } - if (gotOutput) { - AVRational codecTimebase = mVideoStream->codec->time_base; - AVRational streamVideoTimebase = mVideoStream->time_base; - - av_packet_rescale_ts(packet, codecTimebase, streamVideoTimebase); - packet->stream_index = mVideoStream->index; - - av_interleaved_write_frame(mOutputFormatContext, packet); - av_packet_unref(packet); - } - } while (gotOutput); - - if (mShouldRecordAudio) { - - int gotOutput, ret; - do { - - AVPacket* packet = mVideoWorker->mAudioPacket; - - ret = avcodec_encode_audio2(mAudioStream->codec, packet, NULL, &gotOutput); - if (ret < 0) - setLastErrorMessage("Couldn't encode frame to audio"); - - else if (gotOutput) { - AVRational codecTimebase = mAudioStream->codec->time_base; - AVRational streamVideoTimebase = mAudioStream->time_base; - - av_packet_rescale_ts(packet, codecTimebase, streamVideoTimebase); - packet->stream_index = mAudioStream->index; - - av_interleaved_write_frame(mOutputFormatContext, packet); - av_packet_unref(packet); - } - } while (gotOutput); - - } + if (mShouldRecordAudio) + flushStream(mVideoWorker->mAudioPacket, mAudioStream, mOutputFormatContext); av_write_trailer(mOutputFormatContext); - avio_close(mOutputFormatContext->pb); + avcodec_close(mVideoStream->codec); sws_freeContext(mSwsContext); @@ -517,7 +501,13 @@ UBFFmpegVideoEncoderWorker::UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* con } UBFFmpegVideoEncoderWorker::~UBFFmpegVideoEncoderWorker() -{} +{ + if (mVideoPacket) + delete mVideoPacket; + + if (mAudioPacket) + delete mAudioPacket; +} void UBFFmpegVideoEncoderWorker::stopEncoding() { @@ -526,7 +516,7 @@ void UBFFmpegVideoEncoderWorker::stopEncoding() mWaitCondition.wakeAll(); } -void UBFFmpegVideoEncoderWorker::queueFrame(AVFrame* frame) +void UBFFmpegVideoEncoderWorker::queueVideoFrame(AVFrame* frame) { if (frame) { mFrameQueueMutex.lock(); @@ -535,7 +525,7 @@ void UBFFmpegVideoEncoderWorker::queueFrame(AVFrame* frame) } } -void UBFFmpegVideoEncoderWorker::queueAudio(AVFrame* frame) +void UBFFmpegVideoEncoderWorker::queueAudioFrame(AVFrame* frame) { if (frame) { mFrameQueueMutex.lock(); @@ -544,7 +534,6 @@ void UBFFmpegVideoEncoderWorker::queueAudio(AVFrame* frame) } } - /** * The main encoding function. Takes the queued image frames and * assembles them into the video @@ -574,77 +563,13 @@ void UBFFmpegVideoEncoderWorker::runEncoding() void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame() { AVFrame* frame = mImageQueue.dequeue(); - - int gotOutput; - av_init_packet(mVideoPacket); - mVideoPacket->data = NULL; - mVideoPacket->size = 0; - - // qDebug() << "Encoding frame to video. Pts: " << frame->pts << "/" << mController->mVideoTimebase; - - if (avcodec_encode_video2(mController->mVideoStream->codec, mVideoPacket, frame, &gotOutput) < 0) - emit error("Error encoding video frame"); - - if (gotOutput) { - AVRational codecTimebase = mController->mVideoStream->codec->time_base; - AVRational streamVideoTimebase = mController->mVideoStream->time_base; - - - // recalculate the timestamp to match the stream's timebase - av_packet_rescale_ts(mVideoPacket, codecTimebase, streamVideoTimebase); - mVideoPacket->stream_index = mController->mVideoStream->index; - - // qDebug() << "Writing encoded packet to file; pts: " << mVideoPacket->pts << "/" << streamVideoTimebase.den; - - av_interleaved_write_frame(mController->mOutputFormatContext, mVideoPacket); - av_packet_unref(mVideoPacket); - } - - // Duct-tape solution. I assume there's a better way of doing this, but: - // some players like VLC show a black screen until the second frame (which - // can be several seconds after the first one). Simply duplicating the first frame - // seems to solve this problem, and also allows the thumbnail to be generated. - - static bool firstRun = true; - if (firstRun) { - firstRun = false; - frame->pts += 1; - mImageQueue.enqueue(frame); // only works when the queue is empty at this point. todo: clean this up! - } - else - // free the frame - av_frame_free(&frame); + writeFrame(frame, mVideoPacket, mController->mVideoStream, mController->mOutputFormatContext); + av_frame_free(&frame); } void UBFFmpegVideoEncoderWorker::writeLatestAudioFrame() { AVFrame *frame = mAudioQueue.dequeue(); - - int gotOutput, ret; - - av_init_packet(mAudioPacket); - mAudioPacket->data = NULL; - mAudioPacket->size = 0; - - //qDebug() << "Encoding audio frame"; - - ret = avcodec_encode_audio2(mController->mAudioStream->codec, mAudioPacket, frame, &gotOutput); - if (ret < 0) - emit error(QString("Error encoding audio frame: ") + avErrorToQString(ret)); - - else if (gotOutput) { - //qDebug() << "Writing audio frame to stream"; - - AVRational codecTimebase = mController->mAudioStream->codec->time_base; - AVRational streamVideoTimebase = mController->mAudioStream->time_base; - - av_packet_rescale_ts(mAudioPacket, codecTimebase, streamVideoTimebase); - mAudioPacket->stream_index = mController->mAudioStream->index; - - av_interleaved_write_frame(mController->mOutputFormatContext, mAudioPacket); - av_packet_unref(mAudioPacket); - } - + writeFrame(frame, mAudioPacket, mController->mAudioStream, mController->mOutputFormatContext); av_frame_free(&frame); } - diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h index 83f74fd7..9c6b6ab9 100644 --- a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h @@ -26,6 +26,16 @@ extern "C" { class UBFFmpegVideoEncoderWorker; class UBPodcastController; +/** + * This class provides an interface between the podcast controller and the ffmpeg + * back-end. + * It includes all the necessary objects and methods to record video (muxer, audio and + * video streams and encoders, etc) from inputs consisting of raw PCM audio and raw RGBA + * images. + * + * A worker thread is used to encode and write the audio and video on-the-fly. + */ + class UBFFmpegVideoEncoder : public UBAbstractVideoEncoder { Q_OBJECT @@ -117,8 +127,8 @@ public: bool isRunning() { return mIsRunning; } - void queueFrame(AVFrame* frame); - void queueAudio(AVFrame *frame); + void queueVideoFrame(AVFrame* frame); + void queueAudioFrame(AVFrame* frame); public slots: void runEncoding(); @@ -128,7 +138,6 @@ signals: void encodingFinished(); void error(QString message); - private: void writeLatestVideoFrame(); void writeLatestAudioFrame(); From 1e731be171e6faaa6497329ef3379a489d269118 Mon Sep 17 00:00:00 2001 From: Craig Watson Date: Mon, 9 May 2016 13:03:54 +0200 Subject: [PATCH 4/5] Added audio level calculation for the VU meter --- src/podcast/ffmpeg/UBMicrophoneInput.cpp | 85 ++++++++++++++++++++++-- src/podcast/ffmpeg/UBMicrophoneInput.h | 3 + 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/src/podcast/ffmpeg/UBMicrophoneInput.cpp b/src/podcast/ffmpeg/UBMicrophoneInput.cpp index b9c3cf2c..20007409 100644 --- a/src/podcast/ffmpeg/UBMicrophoneInput.cpp +++ b/src/podcast/ffmpeg/UBMicrophoneInput.cpp @@ -50,7 +50,6 @@ int UBMicrophoneInput::sampleFormat() int sampleSize = mAudioFormat.sampleSize(); QAudioFormat::SampleType sampleType = mAudioFormat.sampleType(); - // qDebug() << "Input sample format: " << sampleSize << "bits " << sampleType; switch (sampleType) { case QAudioFormat::Unknown: @@ -61,10 +60,12 @@ int UBMicrophoneInput::sampleFormat() return AV_SAMPLE_FMT_S16; if (sampleSize == 32) return AV_SAMPLE_FMT_S32; + break; case QAudioFormat::UnSignedInt: if (sampleSize == 8) return AV_SAMPLE_FMT_U8; + break; case QAudioFormat::Float: return AV_SAMPLE_FMT_FLT; @@ -72,6 +73,8 @@ int UBMicrophoneInput::sampleFormat() default: return AV_SAMPLE_FMT_NONE; } + + return AV_SAMPLE_FMT_NONE; } QString UBMicrophoneInput::codec() @@ -96,11 +99,14 @@ bool UBMicrophoneInput::init() mAudioFormat = mAudioDeviceInfo.preferredFormat(); mAudioInput = new QAudioInput(mAudioDeviceInfo, mAudioFormat, NULL); - //mAudioInput->setNotifyInterval(100); connect(mAudioInput, SIGNAL(stateChanged(QAudio::State)), this, SLOT(onAudioInputStateChanged(QAudio::State))); + qDebug() << "Input sample format: " << mAudioFormat.sampleSize() << "bit" + << mAudioFormat.sampleType() << "at" << mAudioFormat.sampleRate() << "Hz" + << "; codec: " << mAudioFormat.codec(); + return true; } @@ -159,12 +165,26 @@ void UBMicrophoneInput::setInputDevice(QString name) } +static qint64 uSecsElapsed = 0; void UBMicrophoneInput::onDataReady() { int numBytes = mAudioInput->bytesReady(); - if (numBytes > 0) - emit dataAvailable(mIODevice->read(numBytes)); + uSecsElapsed += mAudioFormat.durationForBytes(numBytes); + + // Only emit data every 100ms + if (uSecsElapsed > 100000) { + uSecsElapsed = 0; + QByteArray data = mIODevice->read(numBytes); + + quint8 level = audioLevel(data); + if (level != mLastAudioLevel) { + mLastAudioLevel = level; + emit audioLevelChanged(level); + } + + emit dataAvailable(data); + } } void UBMicrophoneInput::onAudioInputStateChanged(QAudio::State state) @@ -184,6 +204,63 @@ void UBMicrophoneInput::onAudioInputStateChanged(QAudio::State state) } } +/** + * @brief Calculate the current audio level of an array of samples and return it + * @param data An array of audio samples + * @return A value between 0 and 255 + * + * Audio level is calculated as the RMS (root mean square) of the samples + * in the supplied array. + */ +quint8 UBMicrophoneInput::audioLevel(const QByteArray &data) +{ + int bytesPerSample = mAudioFormat.bytesPerFrame() / mAudioFormat.channelCount(); + + const char * ptr = data.constData(); + double sum = 0; + int n_samples = data.size() / bytesPerSample; + + for (int i(0); i < (data.size() - bytesPerSample); i += bytesPerSample) { + sum += pow(sampleRelativeLevel(ptr + i), 2); + } + + double rms = sqrt(sum/n_samples); + + // The vu meter looks a bit better when the RMS isn't displayed linearly, as perceived sound + // level increases logarithmically. So here RMS can be substituted by something like + // rms^(1/e) + rms = pow(rms, 1./exp(1)); + + return UINT8_MAX * rms; +} + +/** + * @brief Calculate one sample's level relative to its maximum value + * @param sample One sample, in the format specified by mAudioFormat + * @return A double between 0 and 1.0, where 1.0 is the maximum value the sample can take. + */ +double UBMicrophoneInput::sampleRelativeLevel(const char* sample) +{ + QAudioFormat::SampleType type = mAudioFormat.sampleType(); + int sampleSize = mAudioFormat.sampleSize(); + + if (sampleSize == 16 && type == QAudioFormat::SignedInt) + return double(*reinterpret_cast(sample))/INT16_MAX; + + if (sampleSize == 8 && type == QAudioFormat::SignedInt) + return double(*reinterpret_cast(sample))/INT8_MAX; + + if (sampleSize == 16 && type == QAudioFormat::UnSignedInt) + return double(*reinterpret_cast(sample))/UINT16_MAX; + + if (sampleSize == 8 && type == QAudioFormat::UnSignedInt) + return double(*reinterpret_cast(sample))/UINT8_MAX; + + if (type == QAudioFormat::Float) + return (*reinterpret_cast(sample) + 1.0)/2.; + + return -1; +} /** * @brief Return a meaningful error string based on QAudio error codes diff --git a/src/podcast/ffmpeg/UBMicrophoneInput.h b/src/podcast/ffmpeg/UBMicrophoneInput.h index 8ba306d0..cc74f82e 100644 --- a/src/podcast/ffmpeg/UBMicrophoneInput.h +++ b/src/podcast/ffmpeg/UBMicrophoneInput.h @@ -44,6 +44,8 @@ private slots: void onDataReady(); private: + double sampleRelativeLevel(const char* sample); + quint8 audioLevel(const QByteArray& data); QString getErrorString(QAudio::Error errorCode); QAudioInput* mAudioInput; @@ -52,6 +54,7 @@ private: QAudioFormat mAudioFormat; qint64 mSeekPos; + quint8 mLastAudioLevel; }; #endif // UBMICROPHONEINPUT_H From 89bd259d60f18c20d1ec990eaeca9893d88e92fd Mon Sep 17 00:00:00 2001 From: Craig Watson Date: Fri, 13 May 2016 10:50:35 +0200 Subject: [PATCH 5/5] Clean-up; headers added --- src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp | 30 +++- src/podcast/ffmpeg/UBFFmpegVideoEncoder.h | 21 +++ src/podcast/ffmpeg/UBMicrophoneInput.cpp | 177 +++++++++++--------- src/podcast/ffmpeg/UBMicrophoneInput.h | 29 +++- 4 files changed, 163 insertions(+), 94 deletions(-) diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp index 498fbae4..fa6bc0dd 100644 --- a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp @@ -1,3 +1,24 @@ +/* + * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) + * + * This file is part of OpenBoard. + * + * OpenBoard is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3 of the License, + * with a specific linking exception for the OpenSSL project's + * "OpenSSL" library (or with modified versions of it that use the + * same license as the "OpenSSL" library). + * + * OpenBoard is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with OpenBoard. If not, see . + */ + #include "UBFFmpegVideoEncoder.h" //------------------------------------------------------------------------- @@ -223,11 +244,6 @@ bool UBFFmpegVideoEncoder::init() int inChannelCount = mAudioInput->channelCount(); int inSampleRate = mAudioInput->sampleRate(); - int inSampleSize = mAudioInput->sampleSize(); - - qDebug() << "inChannelCount = " << inChannelCount; - qDebug() << "inSampleRate = " << inSampleRate; - qDebug() << "inSampleSize = " << inSampleSize; // Codec @@ -535,8 +551,8 @@ void UBFFmpegVideoEncoderWorker::queueAudioFrame(AVFrame* frame) } /** - * The main encoding function. Takes the queued image frames and - * assembles them into the video + * The main encoding function. Takes the queued frames and + * writes them to the video and audio streams */ void UBFFmpegVideoEncoderWorker::runEncoding() { diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h index 9c6b6ab9..eed8c16b 100644 --- a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h @@ -1,3 +1,24 @@ +/* + * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) + * + * This file is part of OpenBoard. + * + * OpenBoard is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3 of the License, + * with a specific linking exception for the OpenSSL project's + * "OpenSSL" library (or with modified versions of it that use the + * same license as the "OpenSSL" library). + * + * OpenBoard is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with OpenBoard. If not, see . + */ + #ifndef UBFFMPEGVIDEOENCODER_H #define UBFFMPEGVIDEOENCODER_H diff --git a/src/podcast/ffmpeg/UBMicrophoneInput.cpp b/src/podcast/ffmpeg/UBMicrophoneInput.cpp index 20007409..ad17dd49 100644 --- a/src/podcast/ffmpeg/UBMicrophoneInput.cpp +++ b/src/podcast/ffmpeg/UBMicrophoneInput.cpp @@ -1,3 +1,24 @@ +/* + * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) + * + * This file is part of OpenBoard. + * + * OpenBoard is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3 of the License, + * with a specific linking exception for the OpenSSL project's + * "OpenSSL" library (or with modified versions of it that use the + * same license as the "OpenSSL" library). + * + * OpenBoard is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with OpenBoard. If not, see . + */ + #include "UBMicrophoneInput.h" UBMicrophoneInput::UBMicrophoneInput() @@ -13,80 +34,6 @@ UBMicrophoneInput::~UBMicrophoneInput() delete mAudioInput; } -int UBMicrophoneInput::channelCount() -{ - return mAudioFormat.channelCount(); -} - -int UBMicrophoneInput::sampleRate() -{ - return mAudioFormat.sampleRate(); -} - -/* Return the sample size in bits */ -int UBMicrophoneInput::sampleSize() -{ - return mAudioFormat.sampleSize(); -} - -/** Return the sample format in FFMpeg style (AVSampleFormat enum) */ -int UBMicrophoneInput::sampleFormat() -{ - enum AVSampleFormat { - AV_SAMPLE_FMT_NONE = -1, - AV_SAMPLE_FMT_U8, - AV_SAMPLE_FMT_S16, - AV_SAMPLE_FMT_S32, - AV_SAMPLE_FMT_FLT, - AV_SAMPLE_FMT_DBL, - AV_SAMPLE_FMT_U8P, - AV_SAMPLE_FMT_S16P, - AV_SAMPLE_FMT_S32P, - AV_SAMPLE_FMT_FLTP, - AV_SAMPLE_FMT_DBLP, - AV_SAMPLE_FMT_NB - }; - - int sampleSize = mAudioFormat.sampleSize(); - QAudioFormat::SampleType sampleType = mAudioFormat.sampleType(); - - - switch (sampleType) { - case QAudioFormat::Unknown: - return AV_SAMPLE_FMT_NONE; - - case QAudioFormat::SignedInt: - if (sampleSize == 16) - return AV_SAMPLE_FMT_S16; - if (sampleSize == 32) - return AV_SAMPLE_FMT_S32; - break; - - case QAudioFormat::UnSignedInt: - if (sampleSize == 8) - return AV_SAMPLE_FMT_U8; - break; - - case QAudioFormat::Float: - return AV_SAMPLE_FMT_FLT; - - default: - return AV_SAMPLE_FMT_NONE; - } - - return AV_SAMPLE_FMT_NONE; -} - -QString UBMicrophoneInput::codec() -{ - return mAudioFormat.codec(); -} - -qint64 UBMicrophoneInput::processUSecs() const -{ - return mAudioInput->processedUSecs(); -} - bool UBMicrophoneInput::init() { if (mAudioDeviceInfo.isNull()) { @@ -94,15 +41,14 @@ bool UBMicrophoneInput::init() mAudioDeviceInfo = QAudioDeviceInfo::defaultInputDevice(); } - qDebug() << "Input device name: " << mAudioDeviceInfo.deviceName(); - mAudioFormat = mAudioDeviceInfo.preferredFormat(); - mAudioInput = new QAudioInput(mAudioDeviceInfo, mAudioFormat, NULL); connect(mAudioInput, SIGNAL(stateChanged(QAudio::State)), this, SLOT(onAudioInputStateChanged(QAudio::State))); + + qDebug() << "Input device name: " << mAudioDeviceInfo.deviceName(); qDebug() << "Input sample format: " << mAudioFormat.sampleSize() << "bit" << mAudioFormat.sampleType() << "at" << mAudioFormat.sampleRate() << "Hz" << "; codec: " << mAudioFormat.codec(); @@ -112,8 +58,6 @@ bool UBMicrophoneInput::init() void UBMicrophoneInput::start() { - qDebug() << "starting audio input"; - mIODevice = mAudioInput->start(); connect(mIODevice, SIGNAL(readyRead()), @@ -165,6 +109,74 @@ void UBMicrophoneInput::setInputDevice(QString name) } +int UBMicrophoneInput::channelCount() +{ + return mAudioFormat.channelCount(); +} + +int UBMicrophoneInput::sampleRate() +{ + return mAudioFormat.sampleRate(); +} + +/* Return the sample size in bits */ +int UBMicrophoneInput::sampleSize() +{ + return mAudioFormat.sampleSize(); +} + +/** Return the sample format in FFMpeg style (AVSampleFormat enum) */ +int UBMicrophoneInput::sampleFormat() +{ + enum AVSampleFormat { + AV_SAMPLE_FMT_NONE = -1, + AV_SAMPLE_FMT_U8, + AV_SAMPLE_FMT_S16, + AV_SAMPLE_FMT_S32, + AV_SAMPLE_FMT_FLT, + AV_SAMPLE_FMT_DBL, + AV_SAMPLE_FMT_U8P, + AV_SAMPLE_FMT_S16P, + AV_SAMPLE_FMT_S32P, + AV_SAMPLE_FMT_FLTP, + AV_SAMPLE_FMT_DBLP, + AV_SAMPLE_FMT_NB + }; + + int sampleSize = mAudioFormat.sampleSize(); + QAudioFormat::SampleType sampleType = mAudioFormat.sampleType(); + + switch (sampleType) { + case QAudioFormat::Unknown: + return AV_SAMPLE_FMT_NONE; + + case QAudioFormat::SignedInt: + if (sampleSize == 16) + return AV_SAMPLE_FMT_S16; + if (sampleSize == 32) + return AV_SAMPLE_FMT_S32; + break; + + case QAudioFormat::UnSignedInt: + if (sampleSize == 8) + return AV_SAMPLE_FMT_U8; + break; + + case QAudioFormat::Float: + return AV_SAMPLE_FMT_FLT; + + default: + return AV_SAMPLE_FMT_NONE; + } + + return AV_SAMPLE_FMT_NONE; +} + +QString UBMicrophoneInput::codec() +{ + return mAudioFormat.codec(); +} + static qint64 uSecsElapsed = 0; void UBMicrophoneInput::onDataReady() { @@ -189,7 +201,6 @@ void UBMicrophoneInput::onDataReady() void UBMicrophoneInput::onAudioInputStateChanged(QAudio::State state) { - qDebug() << "Audio input state changed to " << state; switch (state) { case QAudio::StoppedState: if (mAudioInput->error() != QAudio::NoError) { @@ -227,8 +238,7 @@ quint8 UBMicrophoneInput::audioLevel(const QByteArray &data) double rms = sqrt(sum/n_samples); // The vu meter looks a bit better when the RMS isn't displayed linearly, as perceived sound - // level increases logarithmically. So here RMS can be substituted by something like - // rms^(1/e) + // level increases logarithmically. So here RMS is substituted by rms^(1/e) rms = pow(rms, 1./exp(1)); return UINT8_MAX * rms; @@ -237,7 +247,8 @@ quint8 UBMicrophoneInput::audioLevel(const QByteArray &data) /** * @brief Calculate one sample's level relative to its maximum value * @param sample One sample, in the format specified by mAudioFormat - * @return A double between 0 and 1.0, where 1.0 is the maximum value the sample can take. + * @return A double between 0 and 1.0, where 1.0 is the maximum value the sample can take, + * or -1 if the value couldn't be calculated. */ double UBMicrophoneInput::sampleRelativeLevel(const char* sample) { diff --git a/src/podcast/ffmpeg/UBMicrophoneInput.h b/src/podcast/ffmpeg/UBMicrophoneInput.h index cc74f82e..b82f5e70 100644 --- a/src/podcast/ffmpeg/UBMicrophoneInput.h +++ b/src/podcast/ffmpeg/UBMicrophoneInput.h @@ -1,3 +1,24 @@ +/* + * Copyright (C) 2015-2016 Département de l'Instruction Publique (DIP-SEM) + * + * This file is part of OpenBoard. + * + * OpenBoard is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, version 3 of the License, + * with a specific linking exception for the OpenSSL project's + * "OpenSSL" library (or with modified versions of it that use the + * same license as the "OpenSSL" library). + * + * OpenBoard is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with OpenBoard. If not, see . + */ + #ifndef UBMICROPHONEINPUT_H #define UBMICROPHONEINPUT_H @@ -5,7 +26,9 @@ #include /** - * @brief The UBMicrophoneInput class captures uncompressed sound from a microphone + * @brief The UBMicrophoneInput class captures uncompressed sound from a microphone. + * + * Audio samples can be read by connecting to the dataAvailable signal. */ class UBMicrophoneInput : public QObject { @@ -28,10 +51,8 @@ public: int sampleFormat(); QString codec(); - qint64 processUSecs() const; - signals: - /// Send the new volume, between 0 and 255 + /// Send the new audio level, between 0 and 255 void audioLevelChanged(quint8 level); /// Emitted when new audio data is available