diff --git a/FFmpegInterop/FFmpegInteropMSS.cpp b/FFmpegInterop/FFmpegInteropMSS.cpp index 035e0d09..838757b7 100644 --- a/FFmpegInterop/FFmpegInteropMSS.cpp +++ b/FFmpegInterop/FFmpegInteropMSS.cpp @@ -27,6 +27,9 @@ #include "shcore.h" #include #include "collection.h" +#include +#include +#include extern "C" { @@ -38,19 +41,28 @@ using namespace FFmpegInterop; using namespace Platform; using namespace Windows::Storage::Streams; using namespace Windows::Media::MediaProperties; +using namespace std; // Size of the buffer when reading a stream const int FILESTREAMBUFFERSZ = 16384; // Mapping of FFMPEG codec types to Windows recognized subtype strings -IMapView^ create_map() -{ - Platform::Collections::Map^ m = ref new Platform::Collections::Map(); - - // Audio codecs - m->Insert(AV_CODEC_ID_OPUS, "OPUS"); - - return m->GetView(); +IMapView^ create_map() +{ + Platform::Collections::Map^ m = ref new Platform::Collections::Map(); + + // Audio codecs + m->Insert(AV_CODEC_ID_OPUS, L"OPUS"); + + // Subtitle codecs + m->Insert(AV_CODEC_ID_ASS, L"SSA"); + m->Insert(AV_CODEC_ID_DVD_SUBTITLE, L"VobSub"); + m->Insert(AV_CODEC_ID_HDMV_PGS_SUBTITLE, L"PGS"); + m->Insert(AV_CODEC_ID_SSA, L"SSA"); + m->Insert(AV_CODEC_ID_SUBRIP, L"SRT"); + m->Insert(AV_CODEC_ID_TEXT, L"SRT"); + + return m->GetView(); } IMapView^ AvCodecMap = create_map(); @@ -83,7 +95,11 @@ FFmpegInteropMSS::FFmpegInteropMSS() , avVideoCodecCtx(nullptr) , audioStreamIndex(AVERROR_STREAM_NOT_FOUND) , videoStreamIndex(AVERROR_STREAM_NOT_FOUND) + , subtitleStreamIndex(AVERROR_STREAM_NOT_FOUND) , thumbnailStreamIndex(AVERROR_STREAM_NOT_FOUND) + , audioStreamSelected(false) + , videoStreamSelected(false) + , subtitleStreamSelected(false) , fileStreamData(nullptr) , fileStreamBuffer(nullptr) { @@ -100,6 +116,7 @@ FFmpegInteropMSS::~FFmpegInteropMSS() { mss->Starting -= startingRequestedToken; mss->SampleRequested -= sampleRequestedToken; + mss->SwitchStreamsRequested -= switchStreamsRequestedToken; mss = nullptr; } @@ -111,6 +128,7 @@ FFmpegInteropMSS::~FFmpegInteropMSS() { m_pReader->SetAudioStream(AVERROR_STREAM_NOT_FOUND, nullptr); m_pReader->SetVideoStream(AVERROR_STREAM_NOT_FOUND, nullptr); + m_pReader->SetSubtitleStream(AVERROR_STREAM_NOT_FOUND, nullptr); m_pReader = nullptr; } @@ -320,10 +338,6 @@ HRESULT FFmpegInteropMSS::InitFFmpegContext(bool forceAudioDecode, bool forceVid if (SUCCEEDED(hr)) { m_pReader = ref new FFmpegReader(avFormatCtx); - if (m_pReader == nullptr) - { - hr = E_OUTOFMEMORY; - } } if (SUCCEEDED(hr)) @@ -368,6 +382,8 @@ HRESULT FFmpegInteropMSS::InitFFmpegContext(bool forceAudioDecode, bool forceVid hr = audioSampleProvider->AllocateResources(); if (SUCCEEDED(hr)) { + audioStreamSelected = true; + audioSampleProvider->EnableStream(); m_pReader->SetAudioStream(audioStreamIndex, audioSampleProvider); } } @@ -388,33 +404,18 @@ HRESULT FFmpegInteropMSS::InitFFmpegContext(bool forceAudioDecode, bool forceVid if (audioStreamIndex != AVERROR_STREAM_NOT_FOUND) { - // allocate decoding parameters - AVCodecParameters* avAudioCodecParams = avcodec_parameters_alloc(); - if (!avAudioCodecParams) - { - hr = E_OUTOFMEMORY; - DebugMessage(L"Could not allocate decoding parameters\n"); - avformat_close_input(&avFormatCtx); - } - - if (avcodec_parameters_copy(avAudioCodecParams, avFormatCtx->streams[audioStreamIndex]->codecpar) < 0) - { - hr = E_FAIL; - avformat_close_input(&avFormatCtx); - } - // Create audio stream descriptor from parameters - hr = CreateAudioStreamDescriptorFromParameters(avAudioCodecParams); + hr = CreateAudioStreamDescriptorFromParameters(avFormatCtx->streams[audioStreamIndex]->codecpar); if (SUCCEEDED(hr)) { hr = audioSampleProvider->AllocateResources(); if (SUCCEEDED(hr)) { + audioStreamSelected = true; + audioSampleProvider->EnableStream(); m_pReader->SetAudioStream(audioStreamIndex, audioSampleProvider); } } - - avcodec_parameters_free(&avAudioCodecParams); } } } @@ -491,6 +492,8 @@ HRESULT FFmpegInteropMSS::InitFFmpegContext(bool forceAudioDecode, bool forceVid hr = videoSampleProvider->AllocateResources(); if (SUCCEEDED(hr)) { + videoStreamSelected = true; + videoSampleProvider->EnableStream(); m_pReader->SetVideoStream(videoStreamIndex, videoSampleProvider); } } @@ -508,66 +511,76 @@ HRESULT FFmpegInteropMSS::InitFFmpegContext(bool forceAudioDecode, bool forceVid if (SUCCEEDED(hr)) { - // Convert media duration from AV_TIME_BASE to TimeSpan unit - mediaDuration = { LONGLONG(avFormatCtx->duration * 10000000 / double(AV_TIME_BASE)) }; - - if (audioStreamDescriptor) + // Get the subtitle stream + for (unsigned int i = 0; i < avFormatCtx->nb_streams; i++) { - if (videoStreamDescriptor) + if (avFormatCtx->streams[i]->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE) { - if (mss) - { - mss->AddStreamDescriptor(videoStreamDescriptor); - mss->AddStreamDescriptor(audioStreamDescriptor); - } - else + if (FAILED(CreateSubtitleStreamDescriptor(avFormatCtx->streams[i]))) { - mss = ref new MediaStreamSource(videoStreamDescriptor, audioStreamDescriptor); + continue; } - } - else - { - if (mss) + + subtitleStreamIndex = i; + + hr = subtitleSampleProvider->AllocateResources(); + if (SUCCEEDED(hr)) { - mss->AddStreamDescriptor(audioStreamDescriptor); + m_pReader->SetSubtitleStream(subtitleStreamIndex, subtitleSampleProvider); } - else + + if (SUCCEEDED(hr)) { - mss = ref new MediaStreamSource(audioStreamDescriptor); + hr = ConvertCodecName(avcodec_get_name(avFormatCtx->streams[i]->codecpar->codec_id), &subtitleCodecName); } + + break; } } - else if (videoStreamDescriptor) + } + + if (SUCCEEDED(hr)) + { + // Create the MSS if one was not provided + if (mss == nullptr) { - if (mss) - { - mss->AddStreamDescriptor(videoStreamDescriptor); - } - else - { - mss = ref new MediaStreamSource(videoStreamDescriptor); - } + mss = ref new MediaStreamSource(nullptr); } - if (mss) + + // Add streams to the MSS + if (videoStreamDescriptor) { - if (mediaDuration.Duration > 0) - { - mss->Duration = mediaDuration; - mss->CanSeek = true; - } - else - { - // Set buffer time to 0 for realtime streaming to reduce latency - mss->BufferTime = { 0 }; - } + mss->AddStreamDescriptor(videoStreamDescriptor); + } + + if (audioStreamDescriptor) + { + mss->AddStreamDescriptor(audioStreamDescriptor); + } - startingRequestedToken = mss->Starting += ref new TypedEventHandler(this, &FFmpegInteropMSS::OnStarting); - sampleRequestedToken = mss->SampleRequested += ref new TypedEventHandler(this, &FFmpegInteropMSS::OnSampleRequested); + if (subtitleStreamDescriptor) + { + mss->AddStreamDescriptor(subtitleStreamDescriptor); + } + + // Set the duration + mediaDuration = { LONGLONG(avFormatCtx->duration * 10000000 / double(AV_TIME_BASE)) }; // Convert media duration from AV_TIME_BASE to TimeSpan unit + + if (mediaDuration.Duration > 0) + { + mss->Duration = mediaDuration; + mss->CanSeek = true; } else { - hr = E_OUTOFMEMORY; + // Set buffer time to 0 for realtime streaming to reduce latency + mss->BufferTime = { 0 }; } + + // Register event handlers + startingRequestedToken = mss->Starting += ref new TypedEventHandler(this, &FFmpegInteropMSS::OnStarting); + sampleRequestedToken = mss->SampleRequested += ref new TypedEventHandler(this, &FFmpegInteropMSS::OnSampleRequested); + switchStreamsRequestedToken = mss->SwitchStreamsRequested += ref new TypedEventHandler(this, &FFmpegInteropMSS::OnSwitchStreamsRequested); } return hr; @@ -659,7 +672,7 @@ HRESULT FFmpegInteropMSS::CreateAudioStreamDescriptor(bool forceAudioDecode) return (audioStreamDescriptor != nullptr && audioSampleProvider != nullptr) ? S_OK : E_OUTOFMEMORY; } -HRESULT FFmpegInteropMSS::CreateAudioStreamDescriptorFromParameters(AVCodecParameters* avCodecParams) +HRESULT FFmpegInteropMSS::CreateAudioStreamDescriptorFromParameters(const AVCodecParameters* avCodecParams) { if (!AvCodecMap->HasKey(avCodecParams->codec_id)) { @@ -669,7 +682,7 @@ HRESULT FFmpegInteropMSS::CreateAudioStreamDescriptorFromParameters(AVCodecParam AudioEncodingProperties^ audioProperties = ref new AudioEncodingProperties(); audioProperties->SampleRate = avCodecParams->sample_rate; audioProperties->ChannelCount = avCodecParams->channels; - audioProperties->Bitrate = avCodecParams->bit_rate; + audioProperties->Bitrate = static_cast(avCodecParams->bit_rate); audioProperties->Subtype = AvCodecMap->Lookup(avCodecParams->codec_id); audioStreamDescriptor = ref new AudioStreamDescriptor(audioProperties); @@ -712,11 +725,13 @@ HRESULT FFmpegInteropMSS::CreateVideoStreamDescriptor(bool forceVideoDecode) videoProperties->Properties->Insert(MF_MT_INTERLACE_MODE, (uint32)_MFVideoInterlaceMode::MFVideoInterlace_MixedInterlaceOrProgressive); } + if (rotateVideo) { Platform::Guid MF_MT_VIDEO_ROTATION(0xC380465D, 0x2271, 0x428C, 0x9B, 0x83, 0xEC, 0xEA, 0x3B, 0x4A, 0x85, 0xC1); videoProperties->Properties->Insert(MF_MT_VIDEO_ROTATION, (uint32)rotationAngle); } + // Detect the correct framerate if (avVideoCodecCtx->framerate.num != 0 || avVideoCodecCtx->framerate.den != 1) { @@ -735,6 +750,50 @@ HRESULT FFmpegInteropMSS::CreateVideoStreamDescriptor(bool forceVideoDecode) return (videoStreamDescriptor != nullptr && videoSampleProvider != nullptr) ? S_OK : E_OUTOFMEMORY; } +HRESULT FFmpegInteropMSS::CreateSubtitleStreamDescriptor(const AVStream* avStream) +{ + _ASSERT(avStream->codecpar->codec_type == AVMEDIA_TYPE_SUBTITLE); + + // Check if we recognize the codec + if (!AvCodecMap->HasKey(avStream->codecpar->codec_id)) + { + return MF_E_INVALIDMEDIATYPE; + } + + // Create encoding properties and set subtype + TimedMetadataEncodingProperties^ subtitleEncProp = ref new TimedMetadataEncodingProperties(); + + subtitleEncProp->Subtype = AvCodecMap->Lookup(avStream->codecpar->codec_id); + + if (avStream->codecpar->extradata != nullptr && avStream->codecpar->extradata_size > 0) + { + subtitleEncProp->SetFormatUserData(ArrayReference(avStream->codecpar->extradata, avStream->codecpar->extradata_size)); + } + + // Create stream descriptor + subtitleStreamDescriptor = ref new TimedMetadataStreamDescriptor(subtitleEncProp); + + const AVDictionaryEntry* avDictEntry = nullptr; + wstring_convert> conv; + + avDictEntry = av_dict_get(avStream->metadata, "title", nullptr, 0); + if (avDictEntry != nullptr) + { + subtitleStreamDescriptor->Name = StringReference(conv.from_bytes(avDictEntry->value).c_str()); + } + + avDictEntry = av_dict_get(avStream->metadata, "language", nullptr, 0); + if (avDictEntry != nullptr) + { + subtitleStreamDescriptor->Language = StringReference(conv.from_bytes(avDictEntry->value).c_str()); + } + + // Create sample provider for this stream + subtitleSampleProvider = ref new MediaSampleProvider(m_pReader, avFormatCtx, nullptr); + + return S_OK; +} + HRESULT FFmpegInteropMSS::ParseOptions(PropertySet^ ffmpegOptions) { HRESULT hr = S_OK; @@ -796,7 +855,7 @@ void FFmpegInteropMSS::OnStarting(MediaStreamSource ^sender, MediaStreamSourceSt // Add deferral // Flush the AudioSampleProvider - if (audioSampleProvider != nullptr) + if (audioSampleProvider != nullptr && audioStreamSelected) { audioSampleProvider->EnableStream(); audioSampleProvider->Flush(); @@ -807,7 +866,7 @@ void FFmpegInteropMSS::OnStarting(MediaStreamSource ^sender, MediaStreamSourceSt } // Flush the VideoSampleProvider - if (videoSampleProvider != nullptr) + if (videoSampleProvider != nullptr && videoStreamSelected) { videoSampleProvider->EnableStream(); videoSampleProvider->Flush(); @@ -816,6 +875,13 @@ void FFmpegInteropMSS::OnStarting(MediaStreamSource ^sender, MediaStreamSourceSt avcodec_flush_buffers(avVideoCodecCtx); } } + + // Flush the subtitleSampleProvider + if (subtitleSampleProvider != nullptr && subtitleStreamSelected) + { + subtitleSampleProvider->EnableStream(); + subtitleSampleProvider->Flush(); + } } } @@ -823,7 +889,7 @@ void FFmpegInteropMSS::OnStarting(MediaStreamSource ^sender, MediaStreamSourceSt } } -void FFmpegInteropMSS::OnSampleRequested(Windows::Media::Core::MediaStreamSource ^sender, MediaStreamSourceSampleRequestedEventArgs ^args) +void FFmpegInteropMSS::OnSampleRequested(MediaStreamSource ^sender, MediaStreamSourceSampleRequestedEventArgs ^args) { mutexGuard.lock(); if (mss != nullptr) @@ -836,6 +902,10 @@ void FFmpegInteropMSS::OnSampleRequested(Windows::Media::Core::MediaStreamSource { args->Request->Sample = videoSampleProvider->GetNextSample(); } + else if (args->Request->StreamDescriptor == subtitleStreamDescriptor && subtitleSampleProvider != nullptr) + { + args->Request->Sample = subtitleSampleProvider->GetNextSample(); + } else { args->Request->Sample = nullptr; @@ -844,6 +914,60 @@ void FFmpegInteropMSS::OnSampleRequested(Windows::Media::Core::MediaStreamSource mutexGuard.unlock(); } +void FFmpegInteropMSS::OnSwitchStreamsRequested(MediaStreamSource ^sender, MediaStreamSourceSwitchStreamsRequestedEventArgs ^args) +{ + mutexGuard.lock(); + if (mss != nullptr) + { + if (args->Request->OldStreamDescriptor != nullptr) + { + if (args->Request->OldStreamDescriptor == audioStreamDescriptor) + { + audioStreamSelected = false; + audioSampleProvider->DisableStream(); + if (avAudioCodecCtx != nullptr) + { + avcodec_flush_buffers(avAudioCodecCtx); + } + } + else if (args->Request->OldStreamDescriptor == videoStreamDescriptor) + { + videoStreamSelected = false; + videoSampleProvider->DisableStream(); + if (avVideoCodecCtx != nullptr) + { + avcodec_flush_buffers(avVideoCodecCtx); + } + } + else if (args->Request->OldStreamDescriptor == subtitleStreamDescriptor) + { + subtitleStreamSelected = false; + subtitleSampleProvider->DisableStream(); + } + } + + if (args->Request->NewStreamDescriptor != nullptr) + { + if (args->Request->NewStreamDescriptor == audioStreamDescriptor) + { + audioStreamSelected = true; + audioSampleProvider->EnableStream(); + } + else if (args->Request->NewStreamDescriptor == videoStreamDescriptor) + { + videoStreamSelected = true; + videoSampleProvider->EnableStream(); + } + else if (args->Request->NewStreamDescriptor == subtitleStreamDescriptor) + { + subtitleStreamSelected = true; + subtitleSampleProvider->EnableStream(); + } + } + } + mutexGuard.unlock(); +} + // Static function to read file stream and pass data to FFmpeg. Credit to Philipp Sch http://www.codeproject.com/Tips/489450/Creating-Custom-FFmpeg-IO-Context static int FileStreamRead(void* ptr, uint8_t* buf, int bufSize) { diff --git a/FFmpegInterop/FFmpegInteropMSS.h b/FFmpegInterop/FFmpegInteropMSS.h index 4d011313..d279b4b9 100644 --- a/FFmpegInterop/FFmpegInteropMSS.h +++ b/FFmpegInterop/FFmpegInteropMSS.h @@ -1,140 +1,164 @@ -//***************************************************************************** -// -// Copyright 2015 Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http ://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//***************************************************************************** - -#pragma once -#include -#include -#include "FFmpegReader.h" -#include "MediaSampleProvider.h" -#include "MediaThumbnailData.h" - -using namespace Platform; -using namespace Windows::Foundation; -using namespace Windows::Foundation::Collections; -using namespace Windows::Media::Core; - -extern "C" -{ -#include -} - -namespace FFmpegInterop -{ - public ref class FFmpegInteropMSS sealed - { - public: - static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromStream(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions, MediaStreamSource^ mss); - static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromStream(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions); - static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromStream(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode); - static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromUri(String^ uri, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions); - static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromUri(String^ uri, bool forceAudioDecode, bool forceVideoDecode); - MediaThumbnailData^ ExtractThumbnail(); - - // Contructor - MediaStreamSource^ GetMediaStreamSource(); - virtual ~FFmpegInteropMSS(); - - // Properties - property AudioStreamDescriptor^ AudioDescriptor - { - AudioStreamDescriptor^ get() - { - return audioStreamDescriptor; - }; - }; - property VideoStreamDescriptor^ VideoDescriptor - { - VideoStreamDescriptor^ get() - { - return videoStreamDescriptor; - }; - }; - property TimeSpan Duration - { - TimeSpan get() - { - return mediaDuration; - }; - }; - property String^ VideoCodecName - { - String^ get() - { - return videoCodecName; - }; - }; - property String^ AudioCodecName - { - String^ get() - { - return audioCodecName; - }; - }; - - void ReleaseFileStream(); - - internal: - int ReadPacket(); - - private: - FFmpegInteropMSS(); - - HRESULT CreateMediaStreamSource(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions, MediaStreamSource^ mss); - HRESULT CreateMediaStreamSource(String^ uri, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions); - HRESULT InitFFmpegContext(bool forceAudioDecode, bool forceVideoDecode); - HRESULT CreateAudioStreamDescriptor(bool forceAudioDecode); - HRESULT CreateAudioStreamDescriptorFromParameters(AVCodecParameters* avCodecParams); - HRESULT CreateVideoStreamDescriptor(bool forceVideoDecode); - HRESULT ConvertCodecName(const char* codecName, String^ *outputCodecName); - HRESULT ParseOptions(PropertySet^ ffmpegOptions); - void OnStarting(MediaStreamSource ^sender, MediaStreamSourceStartingEventArgs ^args); - void OnSampleRequested(MediaStreamSource ^sender, MediaStreamSourceSampleRequestedEventArgs ^args); - - MediaStreamSource^ mss; - EventRegistrationToken startingRequestedToken; - EventRegistrationToken sampleRequestedToken; - - internal: - AVDictionary* avDict; - AVIOContext* avIOCtx; - AVFormatContext* avFormatCtx; - AVCodecContext* avAudioCodecCtx; - AVCodecContext* avVideoCodecCtx; - - private: - AudioStreamDescriptor^ audioStreamDescriptor; - VideoStreamDescriptor^ videoStreamDescriptor; - int audioStreamIndex; - int videoStreamIndex; - int thumbnailStreamIndex; - - bool rotateVideo; - int rotationAngle; - std::recursive_mutex mutexGuard; - - MediaSampleProvider^ audioSampleProvider; - MediaSampleProvider^ videoSampleProvider; - - String^ videoCodecName; - String^ audioCodecName; - TimeSpan mediaDuration; - IStream* fileStreamData; - unsigned char* fileStreamBuffer; - FFmpegReader^ m_pReader; - }; -} +//***************************************************************************** +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http ://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//***************************************************************************** + +#pragma once +#include +#include +#include "FFmpegReader.h" +#include "MediaSampleProvider.h" +#include "MediaThumbnailData.h" + +using namespace Platform; +using namespace Windows::Foundation; +using namespace Windows::Foundation::Collections; +using namespace Windows::Media::Core; + +extern "C" +{ +#include +} + +namespace FFmpegInterop +{ + public ref class FFmpegInteropMSS sealed + { + public: + static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromStream(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions, MediaStreamSource^ mss); + static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromStream(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions); + static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromStream(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode); + static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromUri(String^ uri, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions); + static FFmpegInteropMSS^ CreateFFmpegInteropMSSFromUri(String^ uri, bool forceAudioDecode, bool forceVideoDecode); + MediaThumbnailData^ ExtractThumbnail(); + + // Contructor + MediaStreamSource^ GetMediaStreamSource(); + virtual ~FFmpegInteropMSS(); + + // Properties + property AudioStreamDescriptor^ AudioDescriptor + { + AudioStreamDescriptor^ get() + { + return audioStreamDescriptor; + }; + }; + property VideoStreamDescriptor^ VideoDescriptor + { + VideoStreamDescriptor^ get() + { + return videoStreamDescriptor; + }; + }; + property TimedMetadataStreamDescriptor^ SubtitleDescriptor + { + TimedMetadataStreamDescriptor^ get() + { + return subtitleStreamDescriptor; + }; + }; + property TimeSpan Duration + { + TimeSpan get() + { + return mediaDuration; + }; + }; + property String^ VideoCodecName + { + String^ get() + { + return videoCodecName; + }; + }; + property String^ AudioCodecName + { + String^ get() + { + return audioCodecName; + }; + }; + property String^ SubtitleCodecName + { + String^ get() + { + return subtitleCodecName; + }; + }; + + void ReleaseFileStream(); + + internal: + int ReadPacket(); + + private: + FFmpegInteropMSS(); + + HRESULT CreateMediaStreamSource(IRandomAccessStream^ stream, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions, MediaStreamSource^ mss); + HRESULT CreateMediaStreamSource(String^ uri, bool forceAudioDecode, bool forceVideoDecode, PropertySet^ ffmpegOptions); + HRESULT InitFFmpegContext(bool forceAudioDecode, bool forceVideoDecode); + HRESULT CreateAudioStreamDescriptor(bool forceAudioDecode); + HRESULT CreateAudioStreamDescriptorFromParameters(const AVCodecParameters* avCodecParams); + HRESULT CreateVideoStreamDescriptor(bool forceVideoDecode); + HRESULT CreateSubtitleStreamDescriptor(const AVStream* avStream); + HRESULT ConvertCodecName(const char* codecName, String^ *outputCodecName); + HRESULT ParseOptions(PropertySet^ ffmpegOptions); + void OnStarting(MediaStreamSource ^sender, MediaStreamSourceStartingEventArgs ^args); + void OnSampleRequested(MediaStreamSource ^sender, MediaStreamSourceSampleRequestedEventArgs ^args); + void OnSwitchStreamsRequested(MediaStreamSource ^sender, MediaStreamSourceSwitchStreamsRequestedEventArgs ^args); + + MediaStreamSource^ mss; + EventRegistrationToken startingRequestedToken; + EventRegistrationToken sampleRequestedToken; + EventRegistrationToken switchStreamsRequestedToken; + + internal: + AVDictionary* avDict; + AVIOContext* avIOCtx; + AVFormatContext* avFormatCtx; + AVCodecContext* avAudioCodecCtx; + AVCodecContext* avVideoCodecCtx; + + private: + AudioStreamDescriptor^ audioStreamDescriptor; + VideoStreamDescriptor^ videoStreamDescriptor; + TimedMetadataStreamDescriptor^ subtitleStreamDescriptor; + int audioStreamIndex; + int videoStreamIndex; + int subtitleStreamIndex; + int thumbnailStreamIndex; + bool audioStreamSelected; + bool videoStreamSelected; + bool subtitleStreamSelected; + + bool rotateVideo; + int rotationAngle; + std::recursive_mutex mutexGuard; + + MediaSampleProvider^ audioSampleProvider; + MediaSampleProvider^ videoSampleProvider; + MediaSampleProvider^ subtitleSampleProvider; + + String^ videoCodecName; + String^ audioCodecName; + String^ subtitleCodecName; + TimeSpan mediaDuration; + IStream* fileStreamData; + unsigned char* fileStreamBuffer; + FFmpegReader^ m_pReader; + }; +} diff --git a/FFmpegInterop/FFmpegReader.cpp b/FFmpegInterop/FFmpegReader.cpp index 615aa71e..629e14d2 100644 --- a/FFmpegInterop/FFmpegReader.cpp +++ b/FFmpegInterop/FFmpegReader.cpp @@ -25,6 +25,7 @@ FFmpegReader::FFmpegReader(AVFormatContext* avFormatCtx) : m_pAvFormatCtx(avFormatCtx) , m_audioStreamIndex(AVERROR_STREAM_NOT_FOUND) , m_videoStreamIndex(AVERROR_STREAM_NOT_FOUND) + , m_subtitleStreamIndex(AVERROR_STREAM_NOT_FOUND) { } @@ -57,6 +58,10 @@ int FFmpegReader::ReadPacket() { m_videoSampleProvider->QueuePacket(avPacket); } + else if (avPacket.stream_index == m_subtitleStreamIndex && m_subtitleSampleProvider != nullptr) + { + m_subtitleSampleProvider->QueuePacket(avPacket); + } else { DebugMessage(L"Ignoring unused stream\n"); @@ -86,3 +91,12 @@ void FFmpegReader::SetVideoStream(int videoStreamIndex, MediaSampleProvider^ vid } } +void FFmpegReader::SetSubtitleStream(int subtitleStreamIndex, MediaSampleProvider^ subtitleSampleProvider) +{ + m_subtitleStreamIndex = subtitleStreamIndex; + m_subtitleSampleProvider = subtitleSampleProvider; + if (subtitleSampleProvider != nullptr) + { + subtitleSampleProvider->SetCurrentStreamIndex(m_subtitleStreamIndex); + } +} diff --git a/FFmpegInterop/FFmpegReader.h b/FFmpegInterop/FFmpegReader.h index dfe98ae3..a5dc7648 100644 --- a/FFmpegInterop/FFmpegReader.h +++ b/FFmpegInterop/FFmpegReader.h @@ -30,6 +30,7 @@ namespace FFmpegInterop int ReadPacket(); void SetAudioStream(int audioStreamIndex, MediaSampleProvider^ audioSampleProvider); void SetVideoStream(int videoStreamIndex, MediaSampleProvider^ videoSampleProvider); + void SetSubtitleStream(int subtitleStreamIndex, MediaSampleProvider^ subtitleSampleProvider); internal: FFmpegReader(AVFormatContext* avFormatCtx); @@ -40,5 +41,7 @@ namespace FFmpegInterop int m_audioStreamIndex; MediaSampleProvider^ m_videoSampleProvider; int m_videoStreamIndex; + MediaSampleProvider^ m_subtitleSampleProvider; + int m_subtitleStreamIndex; }; } diff --git a/FFmpegInterop/MediaSampleProvider.cpp b/FFmpegInterop/MediaSampleProvider.cpp index 1d547a45..ceb80f6b 100644 --- a/FFmpegInterop/MediaSampleProvider.cpp +++ b/FFmpegInterop/MediaSampleProvider.cpp @@ -1,168 +1,168 @@ -//***************************************************************************** -// -// Copyright 2015 Microsoft Corporation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http ://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -//***************************************************************************** - -#include "pch.h" -#include "MediaSampleProvider.h" -#include "FFmpegInteropMSS.h" -#include "FFmpegReader.h" - -using namespace FFmpegInterop; - -MediaSampleProvider::MediaSampleProvider( - FFmpegReader^ reader, - AVFormatContext* avFormatCtx, - AVCodecContext* avCodecCtx) - : m_pReader(reader) - , m_pAvFormatCtx(avFormatCtx) - , m_pAvCodecCtx(avCodecCtx) - , m_streamIndex(AVERROR_STREAM_NOT_FOUND) - , m_nextFramePts(AV_NOPTS_VALUE) - , m_isEnabled(true) - , m_isDiscontinuous(false) -{ - DebugMessage(L"MediaSampleProvider\n"); - - m_startOffset = (m_pAvFormatCtx->start_time != AV_NOPTS_VALUE) ? static_cast(m_pAvFormatCtx->start_time * 10000000 / double(AV_TIME_BASE)) : 0; -} - -HRESULT MediaSampleProvider::AllocateResources() -{ - DebugMessage(L"AllocateResources\n"); - return S_OK; -} - -MediaSampleProvider::~MediaSampleProvider() -{ - DebugMessage(L"~MediaSampleProvider\n"); -} - -void MediaSampleProvider::SetCurrentStreamIndex(int streamIndex) -{ - DebugMessage(L"SetCurrentStreamIndex\n"); - if (m_pAvFormatCtx->nb_streams > (unsigned int)streamIndex) - { - m_streamIndex = streamIndex; - } - else - { - m_streamIndex = AVERROR_STREAM_NOT_FOUND; - } -} - -MediaStreamSample^ MediaSampleProvider::GetNextSample() -{ - DebugMessage(L"GetNextSample\n"); - - HRESULT hr = S_OK; - - MediaStreamSample^ sample; - if (m_isEnabled) - { - DataWriter^ dataWriter = ref new DataWriter(); - - LONGLONG pts = 0; - LONGLONG dur = 0; - - hr = GetNextPacket(dataWriter, pts, dur, true); - - if (hr == S_OK) - { - sample = MediaStreamSample::CreateFromBuffer(dataWriter->DetachBuffer(), { pts }); - sample->Duration = { dur }; - sample->Discontinuous = m_isDiscontinuous; - m_isDiscontinuous = false; - } - else +//***************************************************************************** +// +// Copyright 2015 Microsoft Corporation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http ://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +//***************************************************************************** + +#include "pch.h" +#include "MediaSampleProvider.h" +#include "FFmpegInteropMSS.h" +#include "FFmpegReader.h" + +using namespace FFmpegInterop; + +MediaSampleProvider::MediaSampleProvider( + FFmpegReader^ reader, + AVFormatContext* avFormatCtx, + AVCodecContext* avCodecCtx) + : m_pReader(reader) + , m_pAvFormatCtx(avFormatCtx) + , m_pAvCodecCtx(avCodecCtx) + , m_streamIndex(AVERROR_STREAM_NOT_FOUND) + , m_nextFramePts(AV_NOPTS_VALUE) + , m_isEnabled(false) + , m_isDiscontinuous(true) +{ + DebugMessage(L"MediaSampleProvider\n"); + + m_startOffset = (m_pAvFormatCtx->start_time != AV_NOPTS_VALUE) ? static_cast(m_pAvFormatCtx->start_time * 10000000 / double(AV_TIME_BASE)) : 0; +} + +HRESULT MediaSampleProvider::AllocateResources() +{ + DebugMessage(L"AllocateResources\n"); + return S_OK; +} + +MediaSampleProvider::~MediaSampleProvider() +{ + DebugMessage(L"~MediaSampleProvider\n"); +} + +void MediaSampleProvider::SetCurrentStreamIndex(int streamIndex) +{ + DebugMessage(L"SetCurrentStreamIndex\n"); + if (m_pAvFormatCtx->nb_streams > (unsigned int)streamIndex) + { + m_streamIndex = streamIndex; + } + else + { + m_streamIndex = AVERROR_STREAM_NOT_FOUND; + } +} + +MediaStreamSample^ MediaSampleProvider::GetNextSample() +{ + DebugMessage(L"GetNextSample\n"); + + HRESULT hr = S_OK; + + MediaStreamSample^ sample; + if (m_isEnabled) + { + DataWriter^ dataWriter = ref new DataWriter(); + + LONGLONG pts = 0; + LONGLONG dur = 0; + + hr = GetNextPacket(dataWriter, pts, dur, true); + + if (hr == S_OK) { - DebugMessage(L"Too many broken packets - disable stream\n"); + sample = MediaStreamSample::CreateFromBuffer(dataWriter->DetachBuffer(), { pts }); + sample->Duration = { dur }; + sample->Discontinuous = m_isDiscontinuous; + m_isDiscontinuous = false; + } + else if (hr != MF_E_END_OF_STREAM) + { + DebugMessage(L"Too many broken packets - disable stream\n"); DisableStream(); - } - } - - return sample; -} - -HRESULT MediaSampleProvider::WriteAVPacketToStream(DataWriter^ dataWriter, AVPacket* avPacket) -{ - // This is the simplest form of transfer. Copy the packet directly to the stream - // This works for most compressed formats - auto aBuffer = ref new Platform::Array(avPacket->data, avPacket->size); - dataWriter->WriteBytes(aBuffer); - return S_OK; -} - -HRESULT MediaSampleProvider::DecodeAVPacket(DataWriter^ dataWriter, AVPacket *avPacket, int64_t &framePts, int64_t &frameDuration) -{ - // For the simple case of compressed samples, each packet is a sample - if (avPacket != nullptr) - { - frameDuration = avPacket->duration; - if (avPacket->pts != AV_NOPTS_VALUE) - { - framePts = avPacket->pts; - // Set the PTS for the next sample if it doesn't one. - m_nextFramePts = framePts + frameDuration; - } - else - { - if (m_nextFramePts == AV_NOPTS_VALUE) - { - m_nextFramePts = (m_pAvFormatCtx->streams[m_streamIndex]->start_time != AV_NOPTS_VALUE) ? m_pAvFormatCtx->streams[m_streamIndex]->start_time : 0; - } - - framePts = m_nextFramePts; - // Set the PTS for the next sample if it doesn't one. - m_nextFramePts += frameDuration; - } - } - return S_OK; -} - -void MediaSampleProvider::QueuePacket(AVPacket packet) -{ - DebugMessage(L" - QueuePacket\n"); - - if (m_isEnabled) - { - m_packetQueue.push_back(packet); - } - else - { - av_packet_unref(&packet); - } -} - -AVPacket MediaSampleProvider::PopPacket() -{ - DebugMessage(L" - PopPacket\n"); - - AVPacket avPacket; - av_init_packet(&avPacket); - avPacket.data = NULL; - avPacket.size = 0; - - if (!m_packetQueue.empty()) - { - avPacket = m_packetQueue.front(); - m_packetQueue.erase(m_packetQueue.begin()); - } - - return avPacket; + } + } + + return sample; +} + +HRESULT MediaSampleProvider::WriteAVPacketToStream(DataWriter^ dataWriter, AVPacket* avPacket) +{ + // This is the simplest form of transfer. Copy the packet directly to the stream + // This works for most compressed formats + auto aBuffer = ref new Platform::Array(avPacket->data, avPacket->size); + dataWriter->WriteBytes(aBuffer); + return S_OK; +} + +HRESULT MediaSampleProvider::DecodeAVPacket(DataWriter^ dataWriter, AVPacket *avPacket, int64_t &framePts, int64_t &frameDuration) +{ + // For the simple case of compressed samples, each packet is a sample + if (avPacket != nullptr) + { + frameDuration = avPacket->duration; + if (avPacket->pts != AV_NOPTS_VALUE) + { + framePts = avPacket->pts; + // Set the PTS for the next sample if it doesn't one. + m_nextFramePts = framePts + frameDuration; + } + else + { + if (m_nextFramePts == AV_NOPTS_VALUE) + { + m_nextFramePts = (m_pAvFormatCtx->streams[m_streamIndex]->start_time != AV_NOPTS_VALUE) ? m_pAvFormatCtx->streams[m_streamIndex]->start_time : 0; + } + + framePts = m_nextFramePts; + // Set the PTS for the next sample if it doesn't one. + m_nextFramePts += frameDuration; + } + } + return S_OK; +} + +void MediaSampleProvider::QueuePacket(AVPacket packet) +{ + DebugMessage(L" - QueuePacket\n"); + + if (m_isEnabled) + { + m_packetQueue.push_back(packet); + } + else + { + av_packet_unref(&packet); + } +} + +AVPacket MediaSampleProvider::PopPacket() +{ + DebugMessage(L" - PopPacket\n"); + + AVPacket avPacket; + av_init_packet(&avPacket); + avPacket.data = NULL; + avPacket.size = 0; + + if (!m_packetQueue.empty()) + { + avPacket = m_packetQueue.front(); + m_packetQueue.erase(m_packetQueue.begin()); + } + + return avPacket; } HRESULT FFmpegInterop::MediaSampleProvider::GetNextPacket(DataWriter ^ writer, LONGLONG & pts, LONGLONG & dur, bool allowSkip) @@ -170,85 +170,91 @@ HRESULT FFmpegInterop::MediaSampleProvider::GetNextPacket(DataWriter ^ writer, L HRESULT hr = S_OK; AVPacket avPacket; - av_init_packet(&avPacket); - avPacket.data = NULL; - avPacket.size = 0; - - bool frameComplete = false; - bool decodeSuccess = true; - int64_t framePts = 0, frameDuration = 0; - int errorCount = 0; - - while (SUCCEEDED(hr) && !frameComplete) - { - // Continue reading until there is an appropriate packet in the stream - while (m_packetQueue.empty()) - { - if (m_pReader->ReadPacket() < 0) - { - DebugMessage(L"GetNextSample reaching EOF\n"); - hr = E_FAIL; - break; - } - } - - if (!m_packetQueue.empty()) - { - // Pick the packets from the queue one at a time - avPacket = PopPacket(); - framePts = avPacket.pts; - frameDuration = avPacket.duration; - - // Decode the packet if necessary, it will update the presentation time if necessary - hr = DecodeAVPacket(writer, &avPacket, framePts, frameDuration); - frameComplete = (hr == S_OK); - - if (!frameComplete) - { - m_isDiscontinuous = true; - if (allowSkip && errorCount++ < 10) - { - // skip a few broken packets (maybe make this configurable later) - DebugMessage(L"Skipping broken packet\n"); - hr = S_OK; - } - } - } - } - - if (SUCCEEDED(hr)) - { - // Write the packet out - hr = WriteAVPacketToStream(writer, &avPacket); - - pts = LONGLONG(av_q2d(m_pAvFormatCtx->streams[m_streamIndex]->time_base) * 10000000 * framePts) - m_startOffset; - dur = LONGLONG(av_q2d(m_pAvFormatCtx->streams[m_streamIndex]->time_base) * 10000000 * frameDuration); - } - + av_init_packet(&avPacket); + avPacket.data = NULL; + avPacket.size = 0; + + bool frameComplete = false; + bool decodeSuccess = true; + int64_t framePts = 0, frameDuration = 0; + int errorCount = 0; + + while (SUCCEEDED(hr) && !frameComplete) + { + // Continue reading until there is an appropriate packet in the stream + while (m_packetQueue.empty()) + { + int readResult = m_pReader->ReadPacket(); + if (readResult == AVERROR_EOF) + { + DebugMessage(L"GetNextSample reaching EOF\n"); + hr = MF_E_END_OF_STREAM; + break; + } + else if (readResult < 0) + { + hr = E_FAIL; + break; + } + } + + if (!m_packetQueue.empty()) + { + // Pick the packets from the queue one at a time + avPacket = PopPacket(); + framePts = avPacket.pts; + frameDuration = avPacket.duration; + + // Decode the packet if necessary, it will update the presentation time if necessary + hr = DecodeAVPacket(writer, &avPacket, framePts, frameDuration); + frameComplete = (hr == S_OK); + + if (!frameComplete) + { + m_isDiscontinuous = true; + if (allowSkip && errorCount++ < 10) + { + // skip a few broken packets (maybe make this configurable later) + DebugMessage(L"Skipping broken packet\n"); + hr = S_OK; + } + } + } + } + + if (SUCCEEDED(hr)) + { + // Write the packet out + hr = WriteAVPacketToStream(writer, &avPacket); + + pts = LONGLONG(av_q2d(m_pAvFormatCtx->streams[m_streamIndex]->time_base) * 10000000 * framePts) - m_startOffset; + dur = LONGLONG(av_q2d(m_pAvFormatCtx->streams[m_streamIndex]->time_base) * 10000000 * frameDuration); + } + av_packet_unref(&avPacket); return hr; -} - -void MediaSampleProvider::Flush() -{ - DebugMessage(L"Flush\n"); - while (!m_packetQueue.empty()) - { - av_packet_unref(&PopPacket()); - } - m_isDiscontinuous = true; -} - -void MediaSampleProvider::DisableStream() -{ - DebugMessage(L"DisableStream\n"); - Flush(); - m_isEnabled = false; -} - -void MediaSampleProvider::EnableStream() -{ - DebugMessage(L"EnableStream\n"); - m_isEnabled = true; +} + +void MediaSampleProvider::Flush() +{ + DebugMessage(L"Flush\n"); + while (!m_packetQueue.empty()) + { + av_packet_unref(&PopPacket()); + } + m_isDiscontinuous = true; +} + +void MediaSampleProvider::DisableStream() +{ + DebugMessage(L"DisableStream\n"); + Flush(); + m_isEnabled = false; +} + +void MediaSampleProvider::EnableStream() +{ + DebugMessage(L"EnableStream\n"); + m_isEnabled = true; } \ No newline at end of file diff --git a/FFmpegInterop/pch.h b/FFmpegInterop/pch.h index ebc1da6f..943c7ef7 100644 --- a/FFmpegInterop/pch.h +++ b/FFmpegInterop/pch.h @@ -20,6 +20,7 @@ #include #include +#include // Disable debug string output on non-debug build #if !_DEBUG