Clean-up of OS X podcast recording; fixed frames sometimes carrying over from one video to the next

src/podcast/quicktime/UBQuickTimeFile.h

@@ -59,6 +59,14 @@ class UBQuickTimeFile : public QThread
     Q_OBJECT;
 
     public:
+        struct VideoFrame
+        {
+            CVPixelBufferRef buffer;
+            long timestamp;
+        };
+
+        static QWaitCondition frameBufferNotEmpty;
+
         UBQuickTimeFile(QObject * pParent = 0);
         virtual ~UBQuickTimeFile();
 
@@ -69,23 +77,11 @@ class UBQuickTimeFile : public QThread
         void stop();
 
         CVPixelBufferRef newPixelBuffer();
+        void enqueueVideoFrame(VideoFrame frame);
 
         bool isCompressionSessionRunning() { return mCompressionSessionRunning; }
-
         QString lastErrorMessage() const { return mLastErrorMessage; }
 
-        void endSession();
-
-        struct VideoFrame
-        {
-            CVPixelBufferRef buffer;
-            long timestamp;
-        };
-
-        static QQueue<VideoFrame> frameQueue;
-        static QMutex frameQueueMutex;
-        static QWaitCondition frameBufferNotEmpty;
-
     signals:
         void audioLevelChanged(quint8 level);
         void compressionSessionStarted();
@@ -98,44 +94,55 @@ class UBQuickTimeFile : public QThread
         void enqueueAudioBuffer(void* pBuffer, long pLength);
 
     private:
+        QString mLastErrorMessage;
 
-        bool beginSession();
+        // Format information
-        void setLastErrorMessage(const QString& error);
-
-        void appendVideoFrame(CVPixelBufferRef pixelBuffer, long msTimeStamp);
-        bool appendSampleBuffer(CMSampleBufferRef sampleBuffer);
-        
-        QSize mFrameSize;
         QString mVideoFileName;
+        QSize mFrameSize;
+        long mTimeScale;
+        bool mRecordAudio;
+        QString mAudioRecordingDeviceName;
 
+        // Video/audio encoders and associated objects
         AssetWriterPTR mVideoWriter;
-
         AssetWriterInputPTR mVideoWriterInput;
         AssetWriterInputAdaptorPTR mAdaptor;
-
         AssetWriterInputPTR mAudioWriterInput;
 
+        // Audio recorder
         QPointer<UBAudioQueueRecorder> mWaveRecorder;
+        CMAudioFormatDescriptionRef mAudioFormatDescription;
+
+        // Variables used during encoding
         CFAbsoluteTime mStartTime;
+        CMTime mLastFrameTimestamp;
 
-        CMAudioFormatDescriptionRef mAudioFormatDescription;
-        
-        long mTimeScale;
-        bool mRecordAudio;
-        
         volatile bool mShouldStopCompression;
         volatile bool mCompressionSessionRunning;
 
-        QString mLastErrorMessage;
+        // Dispatch queues to handle passing data to the A/V encoders
-        QString mAudioRecordingDeviceName;
-
         dispatch_queue_t mVideoDispatchQueue;
         dispatch_queue_t mAudioDispatchQueue;
 
-        static QQueue<CMSampleBufferRef> audioQueue;
+        // Queues for frames and audio buffers to be encoded
-        static QMutex audioQueueMutex;
+        QQueue<VideoFrame> frameQueue;
+        QQueue<CMSampleBufferRef> audioQueue;
+
+        QMutex frameQueueMutex;
+        QMutex audioQueueMutex;
+        QMutex audioWriterMutex;
+
+
+        // Private functions
+
+        void setLastErrorMessage(const QString& error);
+
+        bool beginSession();
+        void endSession();
+
+        void appendFrameToVideo(CVPixelBufferRef pixelBuffer, long msTimeStamp);
+        bool appendSampleBuffer(CMSampleBufferRef sampleBuffer);
 
-        static QMutex audioWriterMutex;
 };
 
 #endif /* UBQUICKTIMEFILE_H_ */

src/podcast/quicktime/UBQuickTimeFile.mm

@@ -37,15 +37,8 @@
 
 #include "core/memcheck.h"
 
-QQueue<UBQuickTimeFile::VideoFrame> UBQuickTimeFile::frameQueue;
-QMutex UBQuickTimeFile::frameQueueMutex;
 QWaitCondition UBQuickTimeFile::frameBufferNotEmpty;
 
-
-QQueue<CMSampleBufferRef> UBQuickTimeFile::audioQueue;
-QMutex UBQuickTimeFile::audioQueueMutex;
-QMutex UBQuickTimeFile::audioWriterMutex;
-
 UBQuickTimeFile::UBQuickTimeFile(QObject * pParent)
     : QThread(pParent)
     , mVideoWriter(0)
@@ -65,13 +58,14 @@ UBQuickTimeFile::UBQuickTimeFile(QObject * pParent)
 
 UBQuickTimeFile::~UBQuickTimeFile()
 {
-    // destruction of mWaveRecorder is handled by endSession()
-
 }
 
 bool UBQuickTimeFile::init(const QString& pVideoFileName, const QString& pProfileData, int pFramesPerSecond
                 , const QSize& pFrameSize, bool pRecordAudio, const QString& audioRecordingDevice)
 {
+    Q_UNUSED(pProfileData);
+    Q_UNUSED(pFramesPerSecond);
+
     mFrameSize = pFrameSize;
     mVideoFileName = pVideoFileName;
     mRecordAudio = pRecordAudio;
@@ -109,8 +103,9 @@ void UBQuickTimeFile::run()
                 !frameQueue.isEmpty() &&
                 [mVideoWriterInput isReadyForMoreMediaData]) 
             {
+                // in this case the last few frames may be dropped if the queue isn't empty...
                     VideoFrame frame = frameQueue.dequeue();
-                    appendVideoFrame(frame.buffer, frame.timestamp);
+                    appendFrameToVideo(frame.buffer, frame.timestamp);
             }
 
             frameQueueMutex.unlock();
@@ -174,7 +169,6 @@ bool UBQuickTimeFile::beginSession()
                                                 [NSNumber numberWithInt:frameHeight], AVVideoHeightKey,
                                                 nil];
 
-
     mVideoWriterInput = [[AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo
                                             outputSettings:videoSettings] retain];
     NSCParameterAssert(mVideoWriterInput);
@@ -253,7 +247,7 @@ bool UBQuickTimeFile::beginSession()
     [mVideoWriter startSessionAtSourceTime:CMTimeMake(0, mTimeScale)];
 
     mStartTime = CFAbsoluteTimeGetCurrent(); // used for audio timestamp calculation
-
+    mLastFrameTimestamp = CMTimeMake(0, mTimeScale);
 
     return (mVideoWriter != nil) && (mVideoWriterInput != nil) && canStartWriting;
 }
@@ -267,6 +261,8 @@ void UBQuickTimeFile::endSession()
 
 
     [mVideoWriterInput markAsFinished];
+    if (mAudioWriterInput != 0)
+        [mAudioWriterInput markAsFinished];
 
     [mVideoWriter finishWritingWithCompletionHandler:^{
         [mAdaptor release];
@@ -319,32 +315,48 @@ CVPixelBufferRef UBQuickTimeFile::newPixelBuffer()
 {
     CVPixelBufferRef pixelBuffer = 0;
 
-    if(CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, mAdaptor.pixelBufferPool, &pixelBuffer) != kCVReturnSuccess)
+    CVReturn result = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, mAdaptor.pixelBufferPool, &pixelBuffer);
-    {
+
-        setLastErrorMessage("Could not retrieve CV buffer from pool");
+    if (result != kCVReturnSuccess) {
+        setLastErrorMessage("Could not retrieve CV buffer from pool (error " + QString::number(result) + ")");
         return 0;
     }
 
     return pixelBuffer;
 }
 
+void UBQuickTimeFile::enqueueVideoFrame(VideoFrame frame)
+{
+    frameQueueMutex.lock();
+    frameQueue.enqueue(frame);
+    frameQueueMutex.unlock();
+}
 
 /**
  * \brief Add a frame to the pixel buffer adaptor
  * \param pixelBuffer The CVPixelBufferRef (video frame) to add to the movie
  * \param msTimeStamp Timestamp, in milliseconds, of the frame
  */
-void UBQuickTimeFile::appendVideoFrame(CVPixelBufferRef pixelBuffer, long msTimeStamp)
+void UBQuickTimeFile::appendFrameToVideo(CVPixelBufferRef pixelBuffer, long msTimeStamp)
 {
     //qDebug() << "appending video frame";
     CMTime t = CMTimeMake((msTimeStamp * mTimeScale / 1000.0), mTimeScale);
 
-    bool added = [mAdaptor appendPixelBuffer: pixelBuffer
+    // The timestamp must be both valid and larger than the previous frame's timestamp
-                        withPresentationTime: t];
+    if (CMTIME_IS_VALID(t) && CMTimeCompare(t, mLastFrameTimestamp) == 1) {
 
-    if (!added)
+        bool added = [mAdaptor appendPixelBuffer: pixelBuffer
-        setLastErrorMessage(QString("Could not encode frame at time %1").arg(msTimeStamp));
+                            withPresentationTime: t];
+        if (!added)
+            setLastErrorMessage(QString("Could not encode frame at time %1").arg(msTimeStamp));
 
+        mLastFrameTimestamp = t;
+    }
+
+    else {
+        qDebug() << "Frame dropped; timestamp was smaller or equal to previous frame's timestamp of: "
+                 << mLastFrameTimestamp.value << "/" << mLastFrameTimestamp.timescale;
+    }
 
     CVPixelBufferRelease(pixelBuffer);
 }

src/podcast/quicktime/UBQuickTimeVideoEncoder.cpp

@@ -97,15 +97,12 @@ void UBQuickTimeVideoEncoder::compressionFinished()
 
 void UBQuickTimeVideoEncoder::newPixmap(const QImage& pImage, long timestamp)
 {
-        //qDebug() << "New Frame at ms" << timestamp;
+    //qDebug() << "New Frame at ms" << timestamp;
 
-    if(mQuickTimeCompressionSession.isCompressionSessionRunning())
+    if(mQuickTimeCompressionSession.isCompressionSessionRunning()) {
-    {
+        if(mPendingImageFrames.length() > 0) {
-        if(mPendingImageFrames.length() > 0)
+            foreach(ImageFrame frame, mPendingImageFrames) {
-        {
+                encodeFrame(frame.image, frame.timestamp);
-            foreach(ImageFrame frame, mPendingImageFrames)
-            {
-                    encodeFrame(frame.image, frame.timestamp);
             }
 
             mPendingImageFrames.clear();
@@ -115,8 +112,7 @@ void UBQuickTimeVideoEncoder::newPixmap(const QImage& pImage, long timestamp)
 
         UBQuickTimeFile::frameBufferNotEmpty.wakeAll();
     }
-    else
+    else {
-    {
         qDebug() << "queuing frame, compressor not ready";
 
         ImageFrame frame;
@@ -172,9 +168,7 @@ void UBQuickTimeVideoEncoder::encodeFrame(const QImage& pImage, long timestamp)
     videoFrame.buffer = pixelBuffer;
     videoFrame.timestamp = timestamp;
 
-    UBQuickTimeFile::frameQueueMutex.lock();
+    mQuickTimeCompressionSession.enqueueVideoFrame(videoFrame);
-    UBQuickTimeFile::frameQueue.enqueue(videoFrame);
-    UBQuickTimeFile::frameQueueMutex.unlock();
 }