Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression

--- a/content/media/fmp4/MP4Decoder.cpp +++ b/content/media/fmp4/MP4Decoder.cpp @@ -12,16 +12,20 @@ #include "prlog.h" #ifdef XP_WIN #include "mozilla/WindowsVersion.h" #endif #ifdef MOZ_FFMPEG #include "FFmpegRuntimeLinker.h" #endif +#ifdef MOZ_APPLEMEDIA +#include "apple/AppleCMLinker.h" +#include "apple/AppleVTLinker.h" +#endif namespace mozilla { MediaDecoderStateMachine* MP4Decoder::CreateStateMachine() { return new MediaDecoderStateMachine(this, new MP4Reader(this)); } @@ -98,24 +102,50 @@ IsFFmpegAvailable() // If we can link to FFmpeg, then we can almost certainly play H264 and AAC // with it. return FFmpegRuntimeLinker::Link(); #endif } static bool +IsAppleAvailable() +{ +#ifndef MOZ_APPLEMEDIA + // Not the right platform. + return false; +#else + if (!Preferences::GetBool("media.apple.mp4.enabled", false)) { + // Disabled by preference. + return false; + } + // Attempt to load the required frameworks. + bool haveCoreMedia = AppleCMLinker::Link(); + if (!haveCoreMedia) { + return false; + } + bool haveVideoToolbox = AppleVTLinker::Link(); + if (!haveVideoToolbox) { + return false; + } + // All hurdles cleared! + return true; +#endif +} + +static bool HavePlatformMPEGDecoders() { return Preferences::GetBool("media.fragmented-mp4.use-blank-decoder") || #ifdef XP_WIN // We have H.264/AAC platform decoders on Windows Vista and up. IsVistaOrLater() || #endif IsFFmpegAvailable() || + IsAppleAvailable() || // TODO: Other platforms... false; } /* static */ bool MP4Decoder::IsEnabled() {

--- a/content/media/fmp4/PlatformDecoderModule.cpp +++ b/content/media/fmp4/PlatformDecoderModule.cpp @@ -6,16 +6,19 @@ #include "PlatformDecoderModule.h" #ifdef XP_WIN #include "WMFDecoderModule.h" #endif #ifdef MOZ_FFMPEG #include "FFmpegRuntimeLinker.h" #endif +#ifdef MOZ_APPLEMEDIA +#include "AppleDecoderModule.h" +#endif #include "mozilla/Preferences.h" #include "EMEDecoderModule.h" #include "mozilla/CDMProxy.h" #include "SharedThreadPool.h" #include "MediaTaskQueue.h" namespace mozilla { @@ -37,16 +40,19 @@ PlatformDecoderModule::Init() Preferences::AddBoolVarCache(&sUseBlankDecoder, "media.fragmented-mp4.use-blank-decoder"); Preferences::AddBoolVarCache(&sFFmpegDecoderEnabled, "media.fragmented-mp4.ffmpeg.enabled", false); #ifdef XP_WIN WMFDecoderModule::Init(); #endif +#ifdef MOZ_APPLEMEDIA + AppleDecoderModule::Init(); +#endif } class CreateTaskQueueTask : public nsRunnable { public: NS_IMETHOD Run() { MOZ_ASSERT(NS_IsMainThread()); mTaskQueue = new MediaTaskQueue(GetMediaDecodeThreadPool()); return NS_OK; @@ -110,14 +116,19 @@ PlatformDecoderModule::Create() nsAutoPtr<WMFDecoderModule> m(new WMFDecoderModule()); if (NS_SUCCEEDED(m->Startup())) { return m.forget(); } #endif #ifdef MOZ_FFMPEG if (sFFmpegDecoderEnabled) { return FFmpegRuntimeLinker::CreateDecoderModule(); +#endif +#ifdef MOZ_APPLEMEDIA + nsAutoPtr<AppleDecoderModule> m(new AppleDecoderModule()); + if (NS_SUCCEEDED(m->Startup())) { + return m.forget(); } #endif return nullptr; } } // namespace mozilla

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleCMFunctions.h @@ -0,0 +1,12 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Construct references to each of the CoreMedia symbols we use. + +LINK_FUNC(CMVideoFormatDescriptionCreate) +LINK_FUNC(CMBlockBufferCreateWithMemoryBlock) +LINK_FUNC(CMSampleBufferCreate) +LINK_FUNC(CMTimeMake)

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleCMLinker.cpp @@ -0,0 +1,84 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <dlfcn.h> + +#include "AppleCMLinker.h" +#include "nsDebug.h" + +#ifdef PR_LOGGING +PRLogModuleInfo* GetDemuxerLog(); +#define LOG(...) PR_LOG(GetDemuxerLog(), PR_LOG_DEBUG, (__VA_ARGS__)) +#else +#define LOG(...) +#endif + +namespace mozilla { + +AppleCMLinker::LinkStatus +AppleCMLinker::sLinkStatus = LinkStatus_INIT; + +void* AppleCMLinker::sLink = nullptr; +nsrefcnt AppleCMLinker::sRefCount = 0; + +#define LINK_FUNC(func) typeof(func) func; +#include "AppleCMFunctions.h" +#undef LINK_FUNC + +/* static */ bool +AppleCMLinker::Link() +{ + // Bump our reference count every time we're called. + // Add a lock or change the thread assertion if + // you need to call this off the main thread. + MOZ_ASSERT(NS_IsMainThread()); + ++sRefCount; + + if (sLinkStatus) { + return sLinkStatus == LinkStatus_SUCCEEDED; + } + + const char* dlname = + "/System/Library/Frameworks/CoreMedia.framework/CoreMedia"; + if (!(sLink = dlopen(dlname, RTLD_NOW | RTLD_LOCAL))) { + NS_WARNING("Couldn't load CoreMedia framework"); + goto fail; + } + +#define LINK_FUNC(func) \ + func = (typeof(func))dlsym(sLink, #func); \ + if (!func) { \ + NS_WARNING("Couldn't load CoreMedia function " #func ); \ + goto fail; \ + } +#include "AppleCMFunctions.h" +#undef LINK_FUNC + + LOG("Loaded CoreMedia framework."); + sLinkStatus = LinkStatus_SUCCEEDED; + return true; + +fail: + Unlink(); + + sLinkStatus = LinkStatus_FAILED; + return false; +} + +/* static */ void +AppleCMLinker::Unlink() +{ + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(sLink && sRefCount > 0, "Unbalanced Unlink()"); + --sRefCount; + if (sLink && sRefCount < 1) { + LOG("Unlinking CoreMedia framework."); + dlclose(sLink); + sLink = nullptr; + } +} + +} // namespace mozilla

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleCMLinker.h @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef AppleCMLinker_h +#define AppleCMLinker_h + +extern "C" { +#pragma GCC visibility push(default) +#include <CoreMedia/CoreMedia.h> +#pragma GCC visibility pop +} + +namespace mozilla { + +class AppleCMLinker +{ +public: + static bool Link(); + static void Unlink(); + +private: + static void* sLink; + static nsrefcnt sRefCount; + + static enum LinkStatus { + LinkStatus_INIT = 0, + LinkStatus_FAILED, + LinkStatus_SUCCEEDED + } sLinkStatus; +}; + +#define LINK_FUNC(func) extern typeof(func)* func; +#include "AppleCMFunctions.h" +#undef LINK_FUNC + +} // namespace mozilla + +#endif // AppleCMLinker_h

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleDecoderModule.cpp @@ -0,0 +1,92 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "AppleCMLinker.h" +#include "AppleDecoderModule.h" +#include "AppleVTDecoder.h" +#include "AppleVTLinker.h" +#include "mozilla/Preferences.h" +#include "mozilla/DebugOnly.h" + +namespace mozilla { + +extern PlatformDecoderModule* CreateBlankDecoderModule(); + +bool AppleDecoderModule::sIsEnabled = false; + +AppleDecoderModule::AppleDecoderModule() +{ +} + +AppleDecoderModule::~AppleDecoderModule() +{ +} + +/* static */ +void +AppleDecoderModule::Init() +{ + MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); + sIsEnabled = Preferences::GetBool("media.apple.mp4.enabled", false); + if (!sIsEnabled) { + return; + } + + // dlopen CoreMedia.framework if it's available. + sIsEnabled = AppleCMLinker::Link(); + if (!sIsEnabled) { + return; + } + + // dlopen VideoToolbox.framework if it's available. + sIsEnabled = AppleVTLinker::Link(); +} + +nsresult +AppleDecoderModule::Startup() +{ + // We don't have any per-instance initialization to do. + // Check whether ::Init() above succeeded to know if + // we're functional. + if (!sIsEnabled) { + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +nsresult +AppleDecoderModule::Shutdown() +{ + MOZ_ASSERT(NS_IsMainThread(), "Must be on main thread."); + AppleVTLinker::Unlink(); + AppleCMLinker::Unlink(); + + return NS_OK; +} + +MediaDataDecoder* +AppleDecoderModule::CreateH264Decoder(const mp4_demuxer::VideoDecoderConfig& aConfig, + mozilla::layers::LayersBackend aLayersBackend, + mozilla::layers::ImageContainer* aImageContainer, + MediaTaskQueue* aVideoTaskQueue, + MediaDataDecoderCallback* aCallback) +{ + return new AppleVTDecoder(aConfig, aVideoTaskQueue, aCallback, aImageContainer); +} + +MediaDataDecoder* +AppleDecoderModule::CreateAACDecoder(const mp4_demuxer::AudioDecoderConfig& aConfig, + MediaTaskQueue* aAudioTaskQueue, + MediaDataDecoderCallback* aCallback) +{ + NS_WARNING("HACK: using a BlankDecoderModule for AAC"); + if (!mBlankDecoder) { + mBlankDecoder = CreateBlankDecoderModule(); + } + return mBlankDecoder->CreateAACDecoder(aConfig, aAudioTaskQueue, aCallback); +} + +} // namespace mozilla

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleDecoderModule.h @@ -0,0 +1,49 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_AppleDecoderModule_h +#define mozilla_AppleDecoderModule_h + +#include "PlatformDecoderModule.h" + +namespace mozilla { + +class AppleDecoderModule : public PlatformDecoderModule { +public: + AppleDecoderModule(); + virtual ~AppleDecoderModule(); + + // Perform any per-instance initialization. + // Main thread only. + nsresult Startup(); + + // Called when the decoders have shutdown. Main thread only. + // Does this really need to be main thread only???? + virtual nsresult Shutdown() MOZ_OVERRIDE; + + // Decode thread. + virtual MediaDataDecoder* + CreateH264Decoder(const mp4_demuxer::VideoDecoderConfig& aConfig, + mozilla::layers::LayersBackend aLayersBackend, + mozilla::layers::ImageContainer* aImageContainer, + MediaTaskQueue* aVideoTaskQueue, + MediaDataDecoderCallback* aCallback) MOZ_OVERRIDE; + + // Decode thread. + virtual MediaDataDecoder* CreateAACDecoder( + const mp4_demuxer::AudioDecoderConfig& aConfig, + MediaTaskQueue* aAudioTaskQueue, + MediaDataDecoderCallback* aCallback) MOZ_OVERRIDE; + + static void Init(); +private: + static bool sIsEnabled; + nsAutoPtr<PlatformDecoderModule> mBlankDecoder; +}; + +} // namespace mozilla + +#endif // mozilla_AppleDecoderModule_h

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleUtils.cpp @@ -0,0 +1,84 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Utility functions to help with Apple API calls. + +#include <AudioToolbox/AudioToolbox.h> +#include "AppleUtils.h" +#include "prlog.h" + +#ifdef PR_LOGGING +PRLogModuleInfo* GetDemuxerLog(); +#define WARN(...) PR_LOG(GetDemuxerLog(), PR_LOG_WARNING, (__VA_ARGS__)) +#else +#define WARN(...) +#endif + +#define PROPERTY_ID_FORMAT "%c%c%c%c" +#define PROPERTY_ID_PRINT(x) ((x) >> 24), \ + ((x) >> 16) & 0xff, \ + ((x) >> 8) & 0xff, \ + (x) & 0xff + +namespace mozilla { + +nsresult +AppleUtils::GetProperty(AudioFileStreamID aAudioFileStream, + AudioFileStreamPropertyID aPropertyID, + void *aData) +{ + UInt32 size; + Boolean writeable; + OSStatus rv = AudioFileStreamGetPropertyInfo(aAudioFileStream, aPropertyID, + &size, &writeable); + + if (rv) { + WARN("Couldn't get property " PROPERTY_ID_FORMAT "

", + PROPERTY_ID_PRINT(aPropertyID)); + return NS_ERROR_FAILURE; + } + + rv = AudioFileStreamGetProperty(aAudioFileStream, aPropertyID, + &size, aData); + + return NS_OK; +} + +void +AppleUtils::SetCFDict(CFMutableDictionaryRef dict, + const char* key, + const char* value) +{ + // We avoid using the CFSTR macros because there's no way to release those. + AutoCFRelease<CFStringRef> keyRef = + CFStringCreateWithCString(NULL, key, kCFStringEncodingUTF8); + AutoCFRelease<CFStringRef> valueRef = + CFStringCreateWithCString(NULL, value, kCFStringEncodingUTF8); + CFDictionarySetValue(dict, keyRef, valueRef); +} + +void +AppleUtils::SetCFDict(CFMutableDictionaryRef dict, + const char* key, + int32_t value) +{ + AutoCFRelease<CFNumberRef> valueRef = + CFNumberCreate(NULL, kCFNumberSInt32Type, &value); + AutoCFRelease<CFStringRef> keyRef = + CFStringCreateWithCString(NULL, key, kCFStringEncodingUTF8); + CFDictionarySetValue(dict, keyRef, valueRef); +} + +void +AppleUtils::SetCFDict(CFMutableDictionaryRef dict, + const char* key, + bool value) +{ + AutoCFRelease<CFStringRef> keyRef = + CFStringCreateWithCString(NULL, key, kCFStringEncodingUTF8); + CFDictionarySetValue(dict, keyRef, value ? kCFBooleanTrue : kCFBooleanFalse); +} + + +} // namespace mozilla

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleUtils.h @@ -0,0 +1,66 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Utility functions to help with Apple API calls. + +#ifndef mozilla_AppleUtils_h +#define mozilla_AppleUtils_h + +#include <AudioToolbox/AudioToolbox.h> +#include "nsError.h" + +namespace mozilla { + +struct AppleUtils { + // Helper to retrieve properties from AudioFileStream objects. + static nsresult GetProperty(AudioFileStreamID aAudioFileStream, + AudioFileStreamPropertyID aPropertyID, + void *aData); + + // Helper to set a string, string pair on a CFMutableDictionaryRef. + static void SetCFDict(CFMutableDictionaryRef dict, + const char* key, + const char* value); + // Helper to set a string, int32_t pair on a CFMutableDictionaryRef. + static void SetCFDict(CFMutableDictionaryRef dict, + const char* key, + int32_t value); + // Helper to set a string, bool pair on a CFMutableDictionaryRef. + static void SetCFDict(CFMutableDictionaryRef dict, + const char* key, + bool value); +}; + +// Wrapper class to call CFRelease on reference types +// when they go out of scope. +template <class T> +class AutoCFRelease { +public: + AutoCFRelease(T aRef) + : mRef(aRef) + { + } + ~AutoCFRelease() + { + if (mRef) { + CFRelease(mRef); + } + } + // Return the wrapped ref so it can be used as an in parameter. + operator T() + { + return mRef; + } + // Return a pointer to the wrapped ref for use as an out parameter. + T* receive() + { + return &mRef; + } +private: + T mRef; +}; + +} // namespace mozilla + +#endif // mozilla_AppleUtils_h

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleVTDecoder.cpp @@ -0,0 +1,398 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <CoreFoundation/CFString.h> +#include "VideoToolbox/Videotoolbox.h" + +#include "AppleUtils.h" +#include "mozilla/SHA1.h" +#include "mp4_demuxer/DecoderData.h" +#include "MP4Reader.h" +#include "MP4Decoder.h" +#include "nsAutoPtr.h" +#include "nsThreadUtils.h" +#include "AppleVTDecoder.h" +#include "AppleVTLinker.h" +#include "prlog.h" +#include "MediaData.h" +#include "VideoUtils.h" + +#ifdef PR_LOGGING +PRLogModuleInfo* GetDemuxerLog(); +#define LOG(...) PR_LOG(GetDemuxerLog(), PR_LOG_DEBUG, (__VA_ARGS__)) +#define LOG_MEDIA_SHA1 +#else +#define LOG(...) +#endif + +namespace mozilla { + +AppleVTDecoder::AppleVTDecoder(const mp4_demuxer::VideoDecoderConfig& aConfig, + MediaTaskQueue* aVideoTaskQueue, + MediaDataDecoderCallback* aCallback, + layers::ImageContainer* aImageContainer) + : mConfig(aConfig) + , mTaskQueue(aVideoTaskQueue) + , mCallback(aCallback) + , mImageContainer(aImageContainer) + , mFormat(nullptr) + , mSession(nullptr) +{ + MOZ_COUNT_CTOR(AppleVTDecoder); + // TODO: Verify aConfig.mime_type. + LOG("Creating AppleVTDecoder for %dx%d h.264 video", + mConfig.display_width, + mConfig.display_height + ); +} + +AppleVTDecoder::~AppleVTDecoder() +{ + MOZ_COUNT_DTOR(AppleVTDecoder); +} + +nsresult +AppleVTDecoder::Init() +{ + nsresult rv = InitializeSession(); + return rv; +} + +nsresult +AppleVTDecoder::Shutdown() +{ + if (mSession) { + LOG("%s: cleaning up session %p", __func__, mSession); + VTDecompressionSessionInvalidate(mSession); + CFRelease(mSession); + mSession = nullptr; + } + if (mFormat) { + LOG("%s: releasing format %p", __func__, mFormat); + CFRelease(mFormat); + mFormat = nullptr; + } + return NS_OK; +} + +nsresult +AppleVTDecoder::Input(mp4_demuxer::MP4Sample* aSample) +{ + LOG("mp4 input sample %p %lld us %lld pts%s %d bytes dispatched", + aSample, + aSample->duration, + aSample->composition_timestamp, + aSample->is_sync_point ? " keyframe" : "", + aSample->size); + +#ifdef LOG_MEDIA_SHA1 + SHA1Sum hash; + hash.update(aSample->data, aSample->size); + uint8_t digest_buf[SHA1Sum::kHashSize]; + hash.finish(digest_buf); + nsAutoCString digest; + for (size_t i = 0; i < sizeof(digest_buf); i++) { + digest.AppendPrintf("%02x", digest_buf[i]); + } + LOG(" sha1 %s", digest.get()); +#endif // LOG_MEDIA_SHA1 + + mTaskQueue->Dispatch( + NS_NewRunnableMethodWithArg<nsAutoPtr<mp4_demuxer::MP4Sample>>( + this, + &AppleVTDecoder::SubmitFrame, + nsAutoPtr<mp4_demuxer::MP4Sample>(aSample))); + return NS_OK; +} + +nsresult +AppleVTDecoder::Flush() +{ + return Drain(); +} + +nsresult +AppleVTDecoder::Drain() +{ + OSStatus rv = VTDecompressionSessionWaitForAsynchronousFrames(mSession); + if (rv != noErr) { + LOG("Error %d draining frames", rv); + return NS_ERROR_FAILURE; + } + return NS_OK; +} + +// +// Implementation details. +// + +// Context object to hold a copy of sample metadata. +class FrameRef { +public: + Microseconds timestamp; + Microseconds duration; + int64_t byte_offset; + bool is_sync_point; + + explicit FrameRef(mp4_demuxer::MP4Sample* aSample) + { + MOZ_ASSERT(aSample); + timestamp = aSample->composition_timestamp; + duration = aSample->duration; + byte_offset = aSample->byte_offset; + is_sync_point = aSample->is_sync_point; + } +}; + +// Callback passed to the VideoToolbox decoder for returning data. +// This needs to be static because the API takes a C-style pair of +// function and userdata pointers. This validates parameters and +// forwards the decoded image back to an object method. +static void +PlatformCallback(void* decompressionOutputRefCon, + void* sourceFrameRefCon, + OSStatus status, + VTDecodeInfoFlags flags, + CVImageBufferRef image, + CMTime presentationTimeStamp, + CMTime presentationDuration) +{ + AppleVTDecoder* decoder = static_cast<AppleVTDecoder*>(decompressionOutputRefCon); + + LOG("AppleVideoDecoder %s status %d flags %d", __func__, status, flags); + + // Validate our arguments. + if (status != noErr || !image) { + NS_WARNING("VideoToolbox decoder returned no data"); + return; + } + if (flags & kVTDecodeInfo_FrameDropped) { + NS_WARNING(" ...frame dropped..."); + } + MOZ_ASSERT(CFGetTypeID(image) == CVPixelBufferGetTypeID(), + "VideoToolbox returned an unexpected image type"); + + // Forward the data back to an object method which can access + // the correct MP4Reader callback. + FrameRef* frameRef = static_cast<FrameRef*>(sourceFrameRefCon); + decoder->OutputFrame(image, frameRef); +} + +// Copy and return a decoded frame. +nsresult +AppleVTDecoder::OutputFrame(CVPixelBufferRef aImage, + FrameRef* aFrameRef) +{ + size_t width = CVPixelBufferGetWidth(aImage); + size_t height = CVPixelBufferGetHeight(aImage); + LOG(" got decoded frame data... %ux%u %s", width, height, + CVPixelBufferIsPlanar(aImage) ? "planar" : "chunked"); + size_t planes = CVPixelBufferGetPlaneCount(aImage); + for (size_t i = 0; i < planes; ++i) { + size_t stride = CVPixelBufferGetBytesPerRowOfPlane(aImage, i); + LOG(" plane %u %ux%u rowbytes %u", + (unsigned)i, + CVPixelBufferGetWidthOfPlane(aImage, i), + CVPixelBufferGetHeightOfPlane(aImage, i), + (unsigned)stride); + } + + MOZ_ASSERT(planes == 2); + VideoData::YCbCrBuffer buffer; + + // Lock the returned image data. + CVReturn rv = CVPixelBufferLockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); + MOZ_ASSERT(rv == kCVReturnSuccess, "error locking pixel data"); + // Y plane. + buffer.mPlanes[0].mData = + static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 0)); + buffer.mPlanes[0].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 0); + buffer.mPlanes[0].mWidth = width; + buffer.mPlanes[0].mHeight = height; + buffer.mPlanes[0].mOffset = 0; + buffer.mPlanes[0].mSkip = 0; + // Cb plane. + buffer.mPlanes[1].mData = + static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 1)); + buffer.mPlanes[1].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1); + buffer.mPlanes[1].mWidth = (width+1) / 2; + buffer.mPlanes[1].mHeight = (height+1) / 2; + buffer.mPlanes[1].mOffset = 0; + buffer.mPlanes[1].mSkip = 1; + // Cr plane. + buffer.mPlanes[2].mData = + static_cast<uint8_t*>(CVPixelBufferGetBaseAddressOfPlane(aImage, 1)); + buffer.mPlanes[2].mStride = CVPixelBufferGetBytesPerRowOfPlane(aImage, 1); + buffer.mPlanes[2].mWidth = (width+1) / 2; + buffer.mPlanes[2].mHeight = (height+1) / 2; + buffer.mPlanes[2].mOffset = 1; + buffer.mPlanes[2].mSkip = 1; + + // Bounds. + VideoInfo info; + info.mDisplay = nsIntSize(width, height); + info.mHasVideo = true; + gfx::IntRect visible = gfx::IntRect(0, + 0, + mConfig.display_width, + mConfig.display_height); + + // Copy the image data into our own format. + nsAutoPtr<VideoData> data; + data = + VideoData::Create(info, + mImageContainer, + nullptr, + aFrameRef->byte_offset, + aFrameRef->timestamp, + aFrameRef->duration, + buffer, + aFrameRef->is_sync_point, + aFrameRef->timestamp, + visible); + // Unlock the returned image data. + CVPixelBufferUnlockBaseAddress(aImage, kCVPixelBufferLock_ReadOnly); + + mCallback->Output(data.forget()); + delete aFrameRef; + return NS_OK; +} + +// Helper to fill in a timestamp structure. +static CMSampleTimingInfo +TimingInfoFromSample(mp4_demuxer::MP4Sample* aSample) +{ + CMSampleTimingInfo timestamp; + + timestamp.duration = CMTimeMake(aSample->duration, USECS_PER_S); + timestamp.presentationTimeStamp = + CMTimeMake(aSample->composition_timestamp, USECS_PER_S); + // No DTS value available from libstagefright. + timestamp.decodeTimeStamp = + CMTimeMake(aSample->composition_timestamp, USECS_PER_S); + + return timestamp; +} + +nsresult +AppleVTDecoder::SubmitFrame(mp4_demuxer::MP4Sample* aSample) +{ + // For some reason this gives me a double-free error with stagefright. + AutoCFRelease<CMBlockBufferRef> block = nullptr; + AutoCFRelease<CMSampleBufferRef> sample = nullptr; + VTDecodeInfoFlags flags; + OSStatus rv; + + // FIXME: This copies the sample data. I think we can provide + // a custom block source which reuses the aSample buffer. + // But note that there may be a problem keeping the samples + // alive over multiple frames. + rv = CMBlockBufferCreateWithMemoryBlock(NULL // Struct allocator. + ,aSample->data + ,aSample->size + ,kCFAllocatorNull // Block allocator. + ,NULL // Block source. + ,0 // Data offset. + ,aSample->size + ,false + ,block.receive()); + NS_ASSERTION(rv == noErr, "Couldn't create CMBlockBuffer"); + CMSampleTimingInfo timestamp = TimingInfoFromSample(aSample); + rv = CMSampleBufferCreate(NULL, block, true, 0, 0, mFormat, 1, 1, ×tamp, 0, NULL, sample.receive()); + NS_ASSERTION(rv == noErr, "Couldn't create CMSampleBuffer"); + rv = VTDecompressionSessionDecodeFrame(mSession, + sample, + 0, + new FrameRef(aSample), + &flags); + NS_ASSERTION(rv == noErr, "Couldn't pass frame to decoder"); + + // Ask for more data. + if (mTaskQueue->IsEmpty()) { + LOG("AppleVTDecoder task queue empty; requesting more data"); + mCallback->InputExhausted(); + } + + return NS_OK; +} + +nsresult +AppleVTDecoder::InitializeSession() +{ + OSStatus rv; + AutoCFRelease<CFMutableDictionaryRef> extensions = + CFDictionaryCreateMutable(NULL, 0, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + AppleUtils::SetCFDict(extensions, "CVImageBufferChromaLocationBottomField", "left"); + AppleUtils::SetCFDict(extensions, "CVImageBufferChromaLocationTopField", "left"); + AppleUtils::SetCFDict(extensions, "FullRangeVideo", true); + + AutoCFRelease<CFMutableDictionaryRef> atoms = + CFDictionaryCreateMutable(NULL, 0, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + AutoCFRelease<CFDataRef> avc_data = CFDataCreate(NULL, + mConfig.extra_data.begin(), mConfig.extra_data.length()); + +#ifdef LOG_MEDIA_SHA1 + SHA1Sum avc_hash; + avc_hash.update(mConfig.extra_data.begin(), mConfig.extra_data.length()); + uint8_t digest_buf[SHA1Sum::kHashSize]; + avc_hash.finish(digest_buf); + nsAutoCString avc_digest; + for (size_t i = 0; i < sizeof(digest_buf); i++) { + avc_digest.AppendPrintf("%02x", digest_buf[i]); + } + LOG("AVCDecoderConfig %ld bytes sha1 %s", + mConfig.extra_data.length(), avc_digest.get()); +#endif // LOG_MEDIA_SHA1 + + CFDictionarySetValue(atoms, CFSTR("avcC"), avc_data); + CFDictionarySetValue(extensions, CFSTR("SampleDescriptionExtensionAtoms"), atoms); + rv = CMVideoFormatDescriptionCreate(NULL, // Use default allocator. + kCMVideoCodecType_H264, + mConfig.display_width, + mConfig.display_height, + extensions, + &mFormat); + if (rv != noErr) { + NS_ERROR("Couldn't create format description!"); + return NS_ERROR_FAILURE; + } + + // Contruct video decoder selection spec. + AutoCFRelease<CFMutableDictionaryRef> spec = + CFDictionaryCreateMutable(NULL, 0, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + // This key is supported (or ignored) but not declared prior to OSX 10.9. +#if MAC_OS_X_VERSION_MAX_ALLOWED < 1090 + AutoCFRelease<CFStringRef> + kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder = + CFStringCreateWithCString(NULL, "EnableHardwareAcceleratedVideoDecoder", + kCFStringEncodingUTF8); +#endif + CFDictionarySetValue(spec, + kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder, + kCFBooleanTrue); + + VTDecompressionOutputCallbackRecord cb = { PlatformCallback, this }; + rv = VTDecompressionSessionCreate(NULL, // Allocator. + mFormat, + spec, // Video decoder selection. + NULL, // Output video format. + &cb, + &mSession); + if (rv != noErr) { + NS_ERROR("Couldn't create decompression session!"); + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +} // namespace mozilla

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleVTDecoder.h @@ -0,0 +1,56 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef mozilla_AppleVTDecoder_h +#define mozilla_AppleVTDecoder_h + +#include "PlatformDecoderModule.h" +#include "mozilla/RefPtr.h" +#include "mozilla/ReentrantMonitor.h" +#include "nsIThread.h" +#include "VideoToolbox/VideoToolbox.h" + +namespace mozilla { + +class MediaTaskQueue; +class MediaDataDecoderCallback; +namespace layers { + class ImageContainer; +} +class FrameRef; + +class AppleVTDecoder : public MediaDataDecoder { +public: + AppleVTDecoder(const mp4_demuxer::VideoDecoderConfig& aConfig, + MediaTaskQueue* aVideoTaskQueue, + MediaDataDecoderCallback* aCallback, + layers::ImageContainer* aImageContainer); + ~AppleVTDecoder(); + virtual nsresult Init() MOZ_OVERRIDE; + virtual nsresult Input(mp4_demuxer::MP4Sample* aSample) MOZ_OVERRIDE; + virtual nsresult Flush() MOZ_OVERRIDE; + virtual nsresult Drain() MOZ_OVERRIDE; + virtual nsresult Shutdown() MOZ_OVERRIDE; + // Return hook for VideoToolbox callback. + nsresult OutputFrame(CVPixelBufferRef aImage, + FrameRef* frameRef); +private: + const mp4_demuxer::VideoDecoderConfig& mConfig; + RefPtr<MediaTaskQueue> mTaskQueue; + MediaDataDecoderCallback* mCallback; + layers::ImageContainer* mImageContainer; + CMVideoFormatDescriptionRef mFormat; + VTDecompressionSessionRef mSession; + + // Method to pass a frame to VideoToolbox for decoding. + nsresult SubmitFrame(mp4_demuxer::MP4Sample* aSample); + // Method to set up the decompression session. + nsresult InitializeSession(); +}; + +} // namespace mozilla + +#endif // mozilla_AppleVTDecoder_h

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleVTFunctions.h @@ -0,0 +1,12 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Construct references to each of the VideoToolbox symbols we use. + +LINK_FUNC(VTDecompressionSessionCreate) +LINK_FUNC(VTDecompressionSessionDecodeFrame) +LINK_FUNC(VTDecompressionSessionInvalidate) +LINK_FUNC(VTDecompressionSessionWaitForAsynchronousFrames)

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleVTLinker.cpp @@ -0,0 +1,88 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <dlfcn.h> + +#include "AppleVTLinker.h" +#include "nsDebug.h" + +#ifdef PR_LOGGING +PRLogModuleInfo* GetDemuxerLog(); +#define LOG(...) PR_LOG(GetDemuxerLog(), PR_LOG_DEBUG, (__VA_ARGS__)) +#else +#define LOG(...) +#endif + +namespace mozilla { + +AppleVTLinker::LinkStatus +AppleVTLinker::sLinkStatus = LinkStatus_INIT; + +void* AppleVTLinker::sLink = nullptr; +nsrefcnt AppleVTLinker::sRefCount = 0; + +#define LINK_FUNC(func) typeof(func) func; +#include "AppleVTFunctions.h" +#undef LINK_FUNC + +/* static */ bool +AppleVTLinker::Link() +{ + // Bump our reference count every time we're called. + // Add a lock or change the thread assertion if + // you need to call this off the main thread. + MOZ_ASSERT(NS_IsMainThread()); + ++sRefCount; + + if (sLinkStatus) { + return sLinkStatus == LinkStatus_SUCCEEDED; + } + + const char* dlname = + "/System/Library/Frameworks/VideoToolbox.framework/VideoToolbox"; + if (!(sLink = dlopen(dlname, RTLD_NOW | RTLD_LOCAL))) { + NS_WARNING("Couldn't load VideoToolbox framework"); + goto fail; + } + +#define LINK_FUNC(func) \ + func = (typeof(func))dlsym(sLink, #func); \ + if (!func) { \ + NS_WARNING("Couldn't load VideoToolbox function " #func ); \ + goto fail; \ + } +#include "AppleVTFunctions.h" +#undef LINK_FUNC + + LOG("Loaded VideoToolbox framework."); + sLinkStatus = LinkStatus_SUCCEEDED; + return true; + +fail: + Unlink(); + + sLinkStatus = LinkStatus_FAILED; + return false; +} + +/* static */ void +AppleVTLinker::Unlink() +{ + // We'll be called by multiple Decoders, one intantiated for + // each media element. Therefore we receive must maintain a + // reference count to avoidunloading our symbols when other + // instances still need them. + MOZ_ASSERT(NS_IsMainThread()); + MOZ_ASSERT(sLink && sRefCount > 0, "Unbalanced Unlink()"); + --sRefCount; + if (sLink && sRefCount < 1) { + LOG("Unlinking VideoToolbox framework."); + dlclose(sLink); + sLink = nullptr; + } +} + +} // namespace mozilla

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/AppleVTLinker.h @@ -0,0 +1,41 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef AppleVTLinker_h +#define AppleVTLinker_h + +extern "C" { +#pragma GCC visibility push(default) +#include "VideoToolbox/VideoToolbox.h" +#pragma GCC visibility pop +} + +namespace mozilla { + +class AppleVTLinker +{ +public: + static bool Link(); + static void Unlink(); + +private: + static void* sLink; + static nsrefcnt sRefCount; + + static enum LinkStatus { + LinkStatus_INIT = 0, + LinkStatus_FAILED, + LinkStatus_SUCCEEDED + } sLinkStatus; +}; + +#define LINK_FUNC(func) extern typeof(func)* func; +#include "AppleVTFunctions.h" +#undef LINK_FUNC + +} // namespace mozilla + +#endif // AppleVTLinker_h

new file mode 100644 --- /dev/null +++ b/content/media/fmp4/apple/VideoToolbox/VideoToolbox.h @@ -0,0 +1,76 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * * License, v. 2.0. If a copy of the MPL was not distributed with this + * * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +// Stub header for VideoToolbox framework API. +// We include our own copy so we can build on MacOS versions +// where it's not available. + +#ifndef mozilla_VideoToolbox_VideoToolbox_h +#define mozilla_VideoToolbox_VideoToolbox_h + +// CoreMedia is available starting in OS X 10.7, +// so we need to dlopen it as well to run on 10.6, +// but we can depend on the real framework headers at build time. + +#include <CoreMedia/CMBase.h> +#include <CoreFoundation/CoreFoundation.h> +#include <CoreVideo/CVPixelBuffer.h> +#include <CoreMedia/CMSampleBuffer.h> +#include <CoreMedia/CMFormatDescription.h> +#include <CoreMedia/CMTime.h> + +typedef uint32_t VTDecodeFrameFlags; +typedef uint32_t VTDecodeInfoFlags; +enum { + kVTDecodeInfo_Asynchronous = 1UL << 0, + kVTDecodeInfo_FrameDropped = 1UL << 1, +}; + +typedef struct OpaqueVTDecompressionSession* VTDecompressionSessionRef; +typedef void (*VTDecompressionOutputCallback)( + void*, + void*, + OSStatus, + VTDecodeInfoFlags, + CVImageBufferRef, + CMTime, + CMTime +); +typedef struct VTDecompressionOutputCallbackRecord { + VTDecompressionOutputCallback decompressionOutputCallback; + void* decompressionOutputRefCon; +} VTDecompressionOutputCallbackRecord; + +OSStatus +VTDecompressionSessionCreate( + CFAllocatorRef, + CMVideoFormatDescriptionRef, + CFDictionaryRef, + CFDictionaryRef, + const VTDecompressionOutputCallbackRecord*, + VTDecompressionSessionRef* +); + +OSStatus +VTDecompressionSessionDecodeFrame( + VTDecompressionSessionRef, + CMSampleBufferRef, + VTDecodeFrameFlags, + void*, + VTDecodeInfoFlags* +); + +OSStatus +VTDecompressionSessionWaitForAsynchronousFrames( + VTDecompressionSessionRef +); + +void +VTDecompressionSessionInvalidate( + VTDecompressionSessionRef +); + +#endif // mozilla_VideoToolbox_VideoToolbox_h

--- a/content/media/fmp4/moz.build +++ b/content/media/fmp4/moz.build @@ -37,11 +37,23 @@ if CONFIG['MOZ_FFMPEG']: 'ffmpeg/libav53', 'ffmpeg/libav54', 'ffmpeg/libav55', ] LOCAL_INCLUDES += [ 'ffmpeg', ] +if CONFIG['MOZ_APPLEMEDIA']: + EXPORTS += [ + 'apple/AppleDecoderModule.h', + ] + UNIFIED_SOURCES += [ + 'apple/AppleCMLinker.cpp', + 'apple/AppleDecoderModule.cpp', + 'apple/AppleUtils.cpp', + 'apple/AppleVTDecoder.cpp', + 'apple/AppleVTLinker.cpp', + ] + FINAL_LIBRARY = 'xul' FAIL_ON_WARNINGS = True