From a2fb735bbc4938e6f2a0a6681a987511587c57fb Mon Sep 17 00:00:00 2001 From: Craig Watson Date: Thu, 28 Apr 2016 15:58:52 +0200 Subject: [PATCH] Podcasts on Linux: working video, no audio yet --- src/core/UBApplication.cpp | 2 - src/podcast/UBPodcastController.cpp | 4 + src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp | 396 ++++++++++++++++++++ src/podcast/ffmpeg/UBFFmpegVideoEncoder.h | 135 +++++++ src/podcast/podcast.pri | 25 ++ 5 files changed, 560 insertions(+), 2 deletions(-) create mode 100644 src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp create mode 100644 src/podcast/ffmpeg/UBFFmpegVideoEncoder.h diff --git a/src/core/UBApplication.cpp b/src/core/UBApplication.cpp index e364027d..5e205fea 100644 --- a/src/core/UBApplication.cpp +++ b/src/core/UBApplication.cpp @@ -520,10 +520,8 @@ void UBApplication::decorateActionMenu(QAction* action) menu->addSeparator(); -#ifndef Q_OS_LINUX // No Podcast on Linux yet menu->addAction(mainWindow->actionPodcast); mainWindow->actionPodcast->setText(tr("Podcast")); -#endif menu->addSeparator(); menu->addAction(mainWindow->actionQuit); diff --git a/src/podcast/UBPodcastController.cpp b/src/podcast/UBPodcastController.cpp index 598b5135..20a96300 100644 --- a/src/podcast/UBPodcastController.cpp +++ b/src/podcast/UBPodcastController.cpp @@ -64,6 +64,8 @@ #elif defined(Q_OS_OSX) #include "quicktime/UBQuickTimeVideoEncoder.h" #include "quicktime/UBAudioQueueRecorder.h" +#elif defined(Q_OS_LINUX) + #include "ffmpeg/UBFFmpegVideoEncoder.h" #endif #include "core/memcheck.h" @@ -309,6 +311,8 @@ void UBPodcastController::start() mVideoEncoder = new UBWindowsMediaVideoEncoder(this); //deleted on stop #elif defined(Q_OS_OSX) mVideoEncoder = new UBQuickTimeVideoEncoder(this); //deleted on stop +#elif defined(Q_OS_LINUX) + mVideoEncoder = new UBFFmpegVideoEncoder(this); #endif if (mVideoEncoder) diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp new file mode 100644 index 00000000..d2dd361a --- /dev/null +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp @@ -0,0 +1,396 @@ +#include "UBFFmpegVideoEncoder.h" + +// Future proofing +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1) +#define av_frame_alloc avcodec_alloc_frame +#define av_frame_free avcodec_free_frame +#endif + +QString avErrorToQString(int errnum) +{ + char error[AV_ERROR_MAX_STRING_SIZE]; + av_make_error_string(error, AV_ERROR_MAX_STRING_SIZE, errnum); + + return QString(error); +} + +/** + * @brief Constructor for the ffmpeg video encoder + * + * + * This class provides an interface between the screencast controller and the ffmpeg + * back-end. It initializes the audio and video encoders and frees them when done; + * worker threads handle the actual encoding of frames. + * + */ +UBFFmpegVideoEncoder::UBFFmpegVideoEncoder(QObject* parent) + : UBAbstractVideoEncoder(parent) + , mOutputFormatContext(NULL) + , mSwsContext(NULL) + , mFile(NULL) +{ + + mTimebase = 100 * framesPerSecond(); + qDebug() << "timebase: " << mTimebase; + + mVideoEncoderThread = new QThread; + mVideoWorker = new UBFFmpegVideoEncoderWorker(this); + mVideoWorker->moveToThread(mVideoEncoderThread); + + connect(mVideoWorker, SIGNAL(error(QString)), + this, SLOT(setLastErrorMessage(QString))); + + connect(mVideoEncoderThread, SIGNAL(started()), + mVideoWorker, SLOT(runEncoding())); + + connect(mVideoWorker, SIGNAL(encodingFinished()), + mVideoEncoderThread, SLOT(quit())); + + connect(mVideoEncoderThread, SIGNAL(finished()), + this, SLOT(finishEncoding())); +} + +UBFFmpegVideoEncoder::~UBFFmpegVideoEncoder() +{ + if (mVideoWorker) + delete mVideoWorker; + + if (mVideoEncoderThread) + delete mVideoEncoderThread; + +} + +void UBFFmpegVideoEncoder::setLastErrorMessage(const QString& pMessage) +{ + qDebug() << "FFmpeg video encoder:" << pMessage; + mLastErrorMessage = pMessage; +} + +bool UBFFmpegVideoEncoder::start() +{ + bool initialized = init(); + + if (initialized) + mVideoEncoderThread->start(); + + return initialized; +} + +bool UBFFmpegVideoEncoder::stop() +{ + qDebug() << "Video encoder: stop requested"; + + mVideoWorker->stopEncoding(); + + return true; +} + +bool UBFFmpegVideoEncoder::init() +{ + // Initialize ffmpeg lib + av_register_all(); + avcodec_register_all(); + + AVDictionary * options = NULL; + int ret; + + // Output format and context + // -------------------------------------- + + if (avformat_alloc_output_context2(&mOutputFormatContext, NULL, + "mp4", NULL) < 0) + { + setLastErrorMessage("Couldn't allocate video format context"); + return false; + } + + // The default codecs for mp4 are h264 and aac, we use those + + + // Video codec and context + // ------------------------------------- + + AVCodec * videoCodec = avcodec_find_encoder(mOutputFormatContext->oformat->video_codec); + if (!videoCodec) { + setLastErrorMessage("Video codec not found"); + return false; + } + + mVideoStream = avformat_new_stream(mOutputFormatContext, 0); + mVideoStream->time_base = {1, mTimebase}; + + avcodec_get_context_defaults3(mVideoStream->codec, videoCodec); + AVCodecContext* c = avcodec_alloc_context3(videoCodec); + + c->bit_rate = videoBitsPerSecond(); + c->width = videoSize().width(); + c->height = videoSize().height(); + c->time_base = {1, mTimebase}; + c->gop_size = 10; + c->max_b_frames = 0; + c->pix_fmt = AV_PIX_FMT_YUV420P; + + if (mOutputFormatContext->oformat->flags & AVFMT_GLOBALHEADER) + c->flags |= AV_CODEC_FLAG_GLOBAL_HEADER; + + /* + * Supported pixel formats for h264 are: + * AV_PIX_FMT_YUV420P + * AV_PIX_FMT_YUV422P + * AV_PIX_FMT_YUV444P + * AV_PIX_FMT_YUVJ420P + */ + + av_dict_set(&options, "preset", "slow", 0); + av_dict_set(&options, "crf", "20", 0); + + ret = avcodec_open2(c, videoCodec, &options); + + if (ret < 0) { + setLastErrorMessage(QString("Couldn't open video codec: ") + avErrorToQString(ret)); + return false; + } + + mVideoStream->codec = c; + + // Source images are RGB32, and should be converted to YUV for h264 video + mSwsContext = sws_getCachedContext(mSwsContext, + c->width, c->height, AV_PIX_FMT_RGB32, + c->width, c->height, c->pix_fmt, + SWS_BICUBIC, 0, 0, 0); + + // Audio codec and context + // ------------------------------------- + /* + AVCodec * audioCodec = avcodec_find_encoder(mOutputFormatContext->oformat->audio_codec); + mAudioStream = avformat_new_stream(mOutputFormatContext, audioCodec); + */ + + + // Open the output file + ret = avio_open(&(mOutputFormatContext->pb), videoFileName().toStdString().c_str(), AVIO_FLAG_WRITE); + + if (ret < 0) { + setLastErrorMessage(QString("Couldn't open video file for writing: ") + avErrorToQString(ret)); + return false; + } + + // Write stream header + ret = avformat_write_header(mOutputFormatContext, NULL); + + if (ret < 0) { + setLastErrorMessage(QString("Couldn't write header to file: ") + avErrorToQString(ret)); + return false; + } + + return true; +} +void UBFFmpegVideoEncoder::newPixmap(const QImage &pImage, long timestamp) +{ + static bool isFirstFrame = true; + if (isFirstFrame) { + timestamp = 0; + isFirstFrame = false; + } + + if (!mVideoWorker->isRunning()) { + qDebug() << "Encoder worker thread not running. Queuing frame."; + mPendingFrames.enqueue({pImage, timestamp}); + } + + else { + // First send any queued frames, then the latest one + while (!mPendingFrames.isEmpty()) { + AVFrame* avFrame = convertFrame(mPendingFrames.dequeue()); + if (avFrame) + mVideoWorker->queueFrame(avFrame); + } + + // note: if converting the frame turns out to be too slow to do here, it + // can always be done from the worker thread (in thta case, + // the worker's queue would contain ImageFrames rather than AVFrames) + + AVFrame* avFrame = convertFrame({pImage, timestamp}); + if (avFrame) + mVideoWorker->queueFrame(avFrame); + + // signal the worker that frames are available + mVideoWorker->mWaitCondition.wakeAll(); + } +} + +/** Convert a frame consisting of a QImage and timestamp to an AVFrame + * with the right pixel format and PTS + */ +AVFrame* UBFFmpegVideoEncoder::convertFrame(ImageFrame frame) +{ + AVFrame* avFrame = av_frame_alloc(); + + avFrame->format = mVideoStream->codec->pix_fmt; + avFrame->width = mVideoStream->codec->width; + avFrame->height = mVideoStream->codec->height; + avFrame->pts = mTimebase * frame.timestamp / 1000; + + const uchar * rgbImage = frame.image.bits(); + + const int in_linesize[1] = { frame.image.bytesPerLine() }; + + // Allocate the output image + if (av_image_alloc(avFrame->data, avFrame->linesize, mVideoStream->codec->width, + mVideoStream->codec->height, mVideoStream->codec->pix_fmt, 32) < 0) + { + setLastErrorMessage("Couldn't allocate image"); + return NULL; + } + + sws_scale(mSwsContext, + (const uint8_t* const*)&rgbImage, + in_linesize, + 0, + mVideoStream->codec->height, + avFrame->data, + avFrame->linesize); + + return avFrame; +} + +void UBFFmpegVideoEncoder::finishEncoding() +{ + qDebug() << "VideoEncoder::finishEncoding called"; + + // Some frames may not be encoded, so we call avcodec_encode_video2 until they're all done + + int gotOutput; + do { + // TODO: get rid of duplicated code (videoWorker does almost exactly this during encoding) + + AVPacket* packet = mVideoWorker->mPacket; + + if (avcodec_encode_video2(mVideoStream->codec, packet, NULL, &gotOutput) < 0) { + setLastErrorMessage("Couldn't encode frame to video"); + continue; + } + if (gotOutput) { + AVRational codecTimebase = mVideoStream->codec->time_base; + AVRational streamTimebase = mVideoStream->time_base; + + av_packet_rescale_ts(packet, codecTimebase, streamTimebase); + packet->stream_index = mVideoStream->index; + + av_interleaved_write_frame(mOutputFormatContext, packet); + av_packet_unref(packet); + } + } while (gotOutput); + + av_write_trailer(mOutputFormatContext); + + avio_close(mOutputFormatContext->pb); + avcodec_close(mVideoStream->codec); + sws_freeContext(mSwsContext); + avformat_free_context(mOutputFormatContext); + + emit encodingFinished(true); +} + +//------------------------------------------------------------------------- +// Worker +//------------------------------------------------------------------------- + +UBFFmpegVideoEncoderWorker::UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller) + : mController(controller) +{ + mStopRequested = false; + mIsRunning = false; + mPacket = new AVPacket(); +} + +UBFFmpegVideoEncoderWorker::~UBFFmpegVideoEncoderWorker() +{} + +void UBFFmpegVideoEncoderWorker::stopEncoding() +{ + qDebug() << "Video worker: stop requested"; + mStopRequested = true; + mWaitCondition.wakeAll(); +} + +void UBFFmpegVideoEncoderWorker::queueFrame(AVFrame* frame) +{ + mFrameQueueMutex.lock(); + mFrameQueue.enqueue(frame); + mFrameQueueMutex.unlock(); +} + +/** + * The main encoding function. Takes the queued image frames and + * assembles them into the video + */ +void UBFFmpegVideoEncoderWorker::runEncoding() +{ + mIsRunning = true; + + while (!mStopRequested) { + mFrameQueueMutex.lock(); + mWaitCondition.wait(&mFrameQueueMutex); + + while (!mFrameQueue.isEmpty()) { + writeLatestVideoFrame(); + } + + /* + while (!mAudioQueue.isEmpty()) { + writeLatestAudioFrame(); + } + */ + + mFrameQueueMutex.unlock(); + } + + emit encodingFinished(); +} + +void UBFFmpegVideoEncoderWorker::writeLatestVideoFrame() +{ + AVFrame* frame = mFrameQueue.dequeue(); + + int gotOutput; + av_init_packet(mPacket); + mPacket->data = NULL; + mPacket->size = 0; + + // qDebug() << "Encoding frame to video. Pts: " << frame->pts << "/" << mController->mTimebase; + + if (avcodec_encode_video2(mController->mVideoStream->codec, mPacket, frame, &gotOutput) < 0) + emit error("Error encoding frame to video"); + + if (gotOutput) { + AVRational codecTimebase = mController->mVideoStream->codec->time_base; + AVRational streamTimebase = mController->mVideoStream->time_base; + + + // recalculate the timestamp to match the stream's timebase + av_packet_rescale_ts(mPacket, codecTimebase, streamTimebase); + mPacket->stream_index = mController->mVideoStream->index; + + // qDebug() << "Writing encoded packet to file; pts: " << mPacket->pts << "/" << streamTimebase.den; + + av_interleaved_write_frame(mController->mOutputFormatContext, mPacket); + av_packet_unref(mPacket); + } + + // Duct-tape solution. I assume there's a better way of doing this, but: + // some players like VLC show a black screen until the second frame (which + // can be several seconds after the first one). Simply duplicating the first frame + // seems to solve this problem, and also allows the thumbnail to be generated. + + static bool firstRun = true; + if (firstRun) { + firstRun = false; + frame->pts += 1; + mFrameQueue.enqueue(frame); // only works when the queue is empty at this point. todo: clean this up! + } + else + // free the frame + av_frame_free(&frame); +} + diff --git a/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h new file mode 100644 index 00000000..4dacaaf6 --- /dev/null +++ b/src/podcast/ffmpeg/UBFFmpegVideoEncoder.h @@ -0,0 +1,135 @@ +#ifndef UBFFMPEGVIDEOENCODER_H +#define UBFFMPEGVIDEOENCODER_H + +extern "C" { + #include + #include + #include + #include + #include + #include + #include + #include +} + +#include +#include + +#include +#include + +#include "podcast/UBAbstractVideoEncoder.h" + +class UBFFmpegVideoEncoderWorker; +class UBPodcastController; + +class UBFFmpegVideoEncoder : public UBAbstractVideoEncoder +{ + Q_OBJECT + + friend class UBFFmpegVideoEncoderWorker; + +public: + + UBFFmpegVideoEncoder(QObject* parent = NULL); + virtual ~UBFFmpegVideoEncoder(); + + bool start(); + bool stop(); + + void newPixmap(const QImage& pImage, long timestamp); + + QString videoFileExtension() const { return "mp4"; } + + QString lastErrorMessage() { return mLastErrorMessage; } + + void setRecordAudio(bool pRecordAudio) { mShouldRecordAudio = pRecordAudio; } + + +signals: + + void encodingFinished(bool ok); + +private slots: + + void setLastErrorMessage(const QString& pMessage); + void finishEncoding(); + +private: + + struct ImageFrame + { + QImage image; + long timestamp; // unit: ms + }; + + AVFrame* convertFrame(ImageFrame frame); + bool init(); + + // Queue for any pixmap that might be sent before the encoder is ready + QQueue mPendingFrames; + + QString mLastErrorMessage; + bool mShouldRecordAudio; + + QThread* mVideoEncoderThread; + UBFFmpegVideoEncoderWorker* mVideoWorker; + + // Muxer + AVFormatContext* mOutputFormatContext; + int mTimebase; + + // Video + AVStream* mVideoStream; + struct SwsContext * mSwsContext; + + // Audio + AVStream* mAudioStream; + + + FILE * mFile; + +}; + + +class UBFFmpegVideoEncoderWorker : public QObject +{ + Q_OBJECT + + friend class UBFFmpegVideoEncoder; + +public: + UBFFmpegVideoEncoderWorker(UBFFmpegVideoEncoder* controller); + ~UBFFmpegVideoEncoderWorker(); + + bool isRunning() { return mIsRunning; } + + void queueFrame(AVFrame* frame); + +public slots: + void runEncoding(); + void stopEncoding(); + +signals: + void encodingFinished(); + void error(QString message); + + +private: + void writeLatestVideoFrame(); + + UBFFmpegVideoEncoder* mController; + + // std::atomic is C++11. This won't work with msvc2010, so a + // newer compiler must be used if this is to be used on Windows + std::atomic mStopRequested; + std::atomic mIsRunning; + + QQueue mFrameQueue; + QMutex mFrameQueueMutex; + QWaitCondition mWaitCondition; + + AVPacket* mPacket; +}; + +#endif // UBFFMPEGVIDEOENCODER_H diff --git a/src/podcast/podcast.pri b/src/podcast/podcast.pri index af441814..0df7b13e 100644 --- a/src/podcast/podcast.pri +++ b/src/podcast/podcast.pri @@ -33,3 +33,28 @@ macx { OBJECTIVE_SOURCES += src/podcast/quicktime/UBQuickTimeFile.mm } + +linux-g++* { + HEADERS += src/podcast/ffmpeg/UBFFmpegVideoEncoder.h + + SOURCES += src/podcast/ffmpeg/UBFFmpegVideoEncoder.cpp + + + FFMPEG = /opt/ffmpeg + + INCLUDEPATH += $${FFMPEG}/include + DEPENDPATH += /usr/lib/x86_64-linux-gnu + + LIBS += -L $${FFMPEG}/lib -lavformat \ + -L $${FFMPEG}/lib -lavcodec \ + -L $${FFMPEG}/lib -lswscale \ + -L $${FFMPEG}/lib -lavutil \ + -lva-x11 \ + -lva \ + -lxcb-shm \ + -lxcb-xfixes \ + -lxcb-render -lxcb-shape -lxcb -lX11 -lasound -lSDL -lx264 -lpthread -lvpx -lvorbisenc -lvorbis -ltheoraenc -ltheoradec -logg -lopus -lmp3lame -lfreetype -lfdk-aac -lass -llzma -lbz2 -lz -ldl -lswresample -lswscale -lavutil -lm + + + QMAKE_CXXFLAGS += -std=c++11 # move this to OpenBoard.pro when we can use C++11 on all platforms +}