diff --git a/.gitignore b/.gitignore index dd37362..2cc788b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,114 @@ -ipch/* -Debug/* -ARM/* -*/Debug/* -*/ARM/* -*/ipch/* -*/Generated Files/* -*/Bin/* -*/PerfLogs/* -*/obj/* -*.sdf -*.bak +# Build Folders (you can keep bin if you'd like, to store dlls and pdbs) +[Bb]in/ +[Oo]bj/ + +# mstest test results +TestResults + +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files *.suo -*.opensdf *.user -*.pfx +*.sln.docstates + +# Build results +[Dd]ebug/ +[Rr]elease/ +x64/ +*_i.c +*_p.c +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.log +*.vspscc +*.vssscc +.builds + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opensdf +*.sdf + +# Visual Studio profiler +*.psess +*.vsp +*.vspx + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper* + +# Mindbench SASS cache +.sass-cache/ + +# NCrunch +*.ncrunch* +.*crunch*.local.xml + +# Installshield output folder +[Ee]xpress + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish + +# Publish Web Output +*.Publish.xml + +# NuGet Packages Directory +packages + +# Windows Azure Build Output +csx +*.build.csdef + +# Windows Store app package directory +AppPackages/ + +# Others +sql +TestResults +[Tt]est[Rr]esult* +*.Cache +ClientBin +[Ss]tyle[Cc]op.* +~$* +*.dbmdl +Generated_Code #added for RIA/Silverlight projects + +# Backup & report files from converting an old project file to a newer +# Visual Studio version. Backup files are not needed, because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML + +# SQL Server files +App_Data/*.mdf +App_Data/*.ldf + diff --git a/CameraEffectInterface/CameraEffectInterface.vcxproj b/CameraEffectInterface/CameraEffectInterface.vcxproj new file mode 100644 index 0000000..c91edbf --- /dev/null +++ b/CameraEffectInterface/CameraEffectInterface.vcxproj @@ -0,0 +1,136 @@ + + + + + Debug + Win32 + + + Debug + ARM + + + Release + Win32 + + + Release + ARM + + + + {b54d292c-6154-413f-9c4c-d4a08cdb7edf} + CameraEffectInterface + en-US + 11.0 + true + CameraEffectInterface + + + + DynamicLibrary + true + v110_wp80 + + + DynamicLibrary + true + v110_wp80 + + + DynamicLibrary + false + true + v110_wp80 + + + DynamicLibrary + false + true + v110_wp80 + + + + + + + + false + + + + _WINRT_DLL;%(PreprocessorDefinitions) + Use + pch.h + $(WindowsSDK_MetadataPath);$(AdditionalUsingDirectories) + true + + + Console + false + ole32.lib;%(IgnoreSpecificDefaultLibraries) + true + + + + + _WINRT_DLL;NDEBUG;%(PreprocessorDefinitions) + Use + pch.h + $(WindowsSDK_MetadataPath);$(AdditionalUsingDirectories) + true + + + Console + false + ole32.lib;%(IgnoreSpecificDefaultLibraries) + true + + + + + _WINRT_DLL;%(PreprocessorDefinitions) + NotUsing + pch.h + $(WindowsSDK_MetadataPath);$(AdditionalUsingDirectories) + true + + + Console + false + ole32.lib;%(IgnoreSpecificDefaultLibraries) + true + + + + + _WINRT_DLL;NDEBUG;%(PreprocessorDefinitions) + Use + pch.h + $(WindowsSDK_MetadataPath);$(AdditionalUsingDirectories) + true + + + Console + false + ole32.lib;%(IgnoreSpecificDefaultLibraries) + true + + + + + true + false + + + + + + + + + + + + + \ No newline at end of file diff --git a/CameraEffectInterface/ICameraEffect.cpp b/CameraEffectInterface/ICameraEffect.cpp new file mode 100644 index 0000000..9dab23c --- /dev/null +++ b/CameraEffectInterface/ICameraEffect.cpp @@ -0,0 +1,6 @@ +// ICameraEffect.cpp +#include "ICameraEffect.h" + +using namespace CameraEffectInterface; +using namespace Platform; + diff --git a/CameraEffectInterface/ICameraEffect.h b/CameraEffectInterface/ICameraEffect.h new file mode 100644 index 0000000..e83f0c7 --- /dev/null +++ b/CameraEffectInterface/ICameraEffect.h @@ -0,0 +1,51 @@ +#pragma once + +namespace CameraEffectInterface +{ + /// + /// The ICameraEffect interface definition + /// This interface can be implemented either from managed or from native code. + /// + + public interface class ICameraEffect { + + /// + /// The camera device, the effect will poll the preview frames from it + /// + property Windows::Phone::Media::Capture::PhotoCaptureDevice^ CaptureDevice + { + void set( Windows::Phone::Media::Capture::PhotoCaptureDevice^ captureDevice); + } + + /// + /// The buffer where image data is written once the effect has been applied. + /// + property Windows::Storage::Streams::IBuffer^ OutputBuffer + { + void set( Windows::Storage::Streams::IBuffer^ OutputBuffer ); + } + + /// + /// The dimensions of the output buffer + /// + property Windows::Foundation::Size OutputBufferSize + { + void set( Windows::Foundation::Size outputBufferSize); + } + + /// + /// Get a frame from the camera and apply an effect on it + /// + /// A buffer with the camera data with the effect applied + /// A task that completes when effect has been applied + Windows::Foundation::IAsyncAction^ GetNewFrameAndApplyEffect(); + + + /// + /// Change the type of effect + /// + void ChangeEffectType(); + + }; + +} \ No newline at end of file diff --git a/CameraEffectInterface/ICameraEffect.vcxproj.filters b/CameraEffectInterface/ICameraEffect.vcxproj.filters new file mode 100644 index 0000000..3c75cb1 --- /dev/null +++ b/CameraEffectInterface/ICameraEffect.vcxproj.filters @@ -0,0 +1,15 @@ + + + + + 5fd0e509-b6ae-4f29-bd2a-4d2cc10f3aa0 + rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms + + + + + + + + + \ No newline at end of file diff --git a/Native Filter Demo.sln b/Native Filter Demo.sln index cfaa778..1c9ddcf 100644 --- a/Native Filter Demo.sln +++ b/Native Filter Demo.sln @@ -5,6 +5,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NativeFilterDemo", "NativeF EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "NativeComponent", "NativeComponent\NativeComponent.vcxproj", "{ECCD1443-9EAA-4666-BC16-6AA5B5C11BA2}" EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CameraEffectInterface", "CameraEffectInterface\CameraEffectInterface.vcxproj", "{B54D292C-6154-413F-9C4C-D4A08CDB7EDF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -67,6 +69,24 @@ Global {ECCD1443-9EAA-4666-BC16-6AA5B5C11BA2}.Release|Win32.Build.0 = Release|Win32 {ECCD1443-9EAA-4666-BC16-6AA5B5C11BA2}.Release|x86.ActiveCfg = Release|Win32 {ECCD1443-9EAA-4666-BC16-6AA5B5C11BA2}.Release|x86.Build.0 = Release|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|Any CPU.ActiveCfg = Debug|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|ARM.ActiveCfg = Debug|ARM + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|ARM.Build.0 = Debug|ARM + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|Mixed Platforms.ActiveCfg = Debug|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|Mixed Platforms.Build.0 = Debug|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|Win32.ActiveCfg = Debug|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|Win32.Build.0 = Debug|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|x86.ActiveCfg = Debug|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Debug|x86.Build.0 = Debug|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|Any CPU.ActiveCfg = Release|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|ARM.ActiveCfg = Release|ARM + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|ARM.Build.0 = Release|ARM + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|Mixed Platforms.ActiveCfg = Release|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|Mixed Platforms.Build.0 = Release|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|Win32.ActiveCfg = Release|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|Win32.Build.0 = Release|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|x86.ActiveCfg = Release|Win32 + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF}.Release|x86.Build.0 = Release|Win32 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/NativeComponent/NativeComponent.cpp b/NativeComponent/NativeComponent.cpp index c3ea1fb..0624458 100644 --- a/NativeComponent/NativeComponent.cpp +++ b/NativeComponent/NativeComponent.cpp @@ -1,8 +1,9 @@ // NativeComponent.cpp #include "pch.h" +#include #include -#include "NativeComponent.h" +#include #if defined(_M_ARM) #include #endif @@ -10,46 +11,77 @@ using namespace NativeComponent; using namespace Platform; using namespace Windows::Phone::Media::Capture; +using namespace Windows::Foundation; +using namespace Windows::Storage::Streams; +using namespace Microsoft::WRL; WindowsPhoneRuntimeComponent::WindowsPhoneRuntimeComponent() { +} +void WindowsPhoneRuntimeComponent::CaptureDevice::set (PhotoCaptureDevice^ device) +{ + m_camera = device; + Windows::Foundation::Size cameraFrameResolution = device->PreviewResolution; + int numberOfPixels = int(cameraFrameResolution.Width) * int (cameraFrameResolution.Height); + m_cameraPreviewBuffer = ref new Array(numberOfPixels); } +void WindowsPhoneRuntimeComponent::OutputBuffer::set (Windows::Storage::Streams::IBuffer^ outputBuffer) +{ + // Com magic to retrieve the pointer to the pixel buffer. + Object^ obj = outputBuffer; + ComPtr insp(reinterpret_cast(obj)); + ComPtr bufferByteAccess; + ThrowIfFailed(insp.As(&bufferByteAccess)); + m_pixelsBuffer = nullptr; + ThrowIfFailed(bufferByteAccess->Buffer(&m_pixelsBuffer)); +} -void WindowsPhoneRuntimeComponent::Initialize(Windows::Phone::Media::Capture::PhotoCaptureDevice^ captureDevice) +void WindowsPhoneRuntimeComponent::OutputBufferSize::set (Windows::Foundation::Size bufferSize) { - m_camera = captureDevice; - Windows::Foundation::Size viewfinderResolution = m_camera->PreviewResolution; + m_outputBufferSize = bufferSize; +} - m_processingFrame = false; +IAsyncAction^ WindowsPhoneRuntimeComponent::GetNewFrameAndApplyEffect() +{ + return concurrency::create_async([this](){ -} + m_camera->GetPreviewBufferArgb(m_cameraPreviewBuffer); +#if defined(_M_ARM) + ConvertToGrayNeon(m_cameraPreviewBuffer,m_pixelsBuffer); +#else + ConvertToGrayOriginal(m_cameraPreviewBuffer,m_pixelsBuffer); +#endif -void WindowsPhoneRuntimeComponent::NewViewfinderFrame( Platform::WriteOnlyArray^ inputBuffer, - Platform::WriteOnlyArray^ outputBuffer) -{ - m_camera->GetPreviewBufferArgb(inputBuffer); - #if defined(_M_ARM) - ConvertToGrayNeon(inputBuffer,outputBuffer ); - #else - ConvertToGrayOriginal(inputBuffer, outputBuffer); - #endif - - + }); } +void WindowsPhoneRuntimeComponent::ChangeEffectType() +{ +} // The gray convertion and its NEON optimization is copied from http://hilbert-space.de/?p=22 void WindowsPhoneRuntimeComponent::ConvertToGrayOriginal( Platform::WriteOnlyArray^ inputBuffer, - Platform::WriteOnlyArray^ outputBuffer) -{ uint8 * src = (uint8 *) inputBuffer->Data; - uint8 * dest = (uint8 *) outputBuffer->Data; + byte * outputBuffer) +{ + uint8 * src = (uint8 *) inputBuffer->Data; + uint8 * dest = (uint8 *) outputBuffer; ConvertToGrayOriginal(src, dest, inputBuffer->Length); } + +inline void WindowsPhoneRuntimeComponent::ThrowIfFailed(HRESULT hr) +{ + if (FAILED(hr)) + { + // Set a breakpoint on this line to catch Win32 API errors. + throw Platform::Exception::CreateException(hr); + } +} + void WindowsPhoneRuntimeComponent::ConvertToGrayOriginal( uint8 * src, uint8* dest, int length) { int i; @@ -77,10 +109,10 @@ void WindowsPhoneRuntimeComponent::ConvertToGrayOriginal( uint8 * src, uint8* de // For a good introduction to NEON: http://www.stanford.edu/class/ee282/10_handouts/lect.10.arm_soc.pdf #if defined(_M_ARM) void WindowsPhoneRuntimeComponent::ConvertToGrayNeon( Platform::WriteOnlyArray^ inputBuffer, - Platform::WriteOnlyArray^ outputBuffer) + byte * outputBuffer) { uint8 * src = (uint8 *) inputBuffer->Data; - uint8 * dest = (uint8 *) outputBuffer->Data; + uint8 * dest = (uint8 *) outputBuffer; int n = inputBuffer->Length; @@ -88,7 +120,7 @@ void WindowsPhoneRuntimeComponent::ConvertToGrayNeon( Platform::WriteOnlyArray^ inputBuffer, - Platform::WriteOnlyArray^ outputBuffer); + virtual property Windows::Phone::Media::Capture::PhotoCaptureDevice^ CaptureDevice + { + void set( Windows::Phone::Media::Capture::PhotoCaptureDevice^ ); + } + + virtual property Windows::Storage::Streams::IBuffer^ OutputBuffer + { + void set( Windows::Storage::Streams::IBuffer^ ); + } + + virtual property Windows::Foundation::Size OutputBufferSize + { + void set( Windows::Foundation::Size ); + } + + virtual Windows::Foundation::IAsyncAction^ GetNewFrameAndApplyEffect(); + + virtual void ChangeEffectType(); private: void ConvertToGrayOriginal( Platform::WriteOnlyArray^ inputBuffer, - Platform::WriteOnlyArray^ outputBuffer); + byte * outputBuffer); void ConvertToGrayOriginal( uint8 * src, uint8* dest, int length); #if defined(_M_ARM) void ConvertToGrayNeon ( Platform::WriteOnlyArray^ inputBuffer, - Platform::WriteOnlyArray^ outputBuffer); + byte * outputBuffer); #endif - + void ThrowIfFailed(HRESULT hr); Windows::Phone::Media::Capture::PhotoCaptureDevice^ m_camera; + Platform::Array^ m_cameraPreviewBuffer; + Windows::Foundation::Size m_outputBufferSize; + byte* m_pixelsBuffer; bool m_processingFrame; }; } \ No newline at end of file diff --git a/NativeComponent/NativeComponent.vcxproj b/NativeComponent/NativeComponent.vcxproj index dd960c6..369a21b 100644 --- a/NativeComponent/NativeComponent.vcxproj +++ b/NativeComponent/NativeComponent.vcxproj @@ -18,7 +18,6 @@ ARM - {eccd1443-9eaa-4666-bc16-6aa5b5c11ba2} NativeComponent @@ -26,9 +25,7 @@ 11.0 true - - DynamicLibrary true @@ -51,19 +48,14 @@ true v110_wp80 - - - - false - _WINRT_DLL;%(PreprocessorDefinitions) @@ -79,7 +71,6 @@ true - _WINRT_DLL;NDEBUG;%(PreprocessorDefinitions) @@ -95,7 +86,6 @@ true - _WINRT_DLL;%(PreprocessorDefinitions) @@ -111,7 +101,6 @@ true - _WINRT_DLL;NDEBUG;%(PreprocessorDefinitions) @@ -127,14 +116,12 @@ true - true false - @@ -145,11 +132,13 @@ Create - + + + {b54d292c-6154-413f-9c4c-d4a08cdb7edf} + + - - - + \ No newline at end of file diff --git a/NativeComponent/NativeComponent.vcxproj.filters b/NativeComponent/NativeComponent.vcxproj.filters index 1d26f49..11449f2 100644 --- a/NativeComponent/NativeComponent.vcxproj.filters +++ b/NativeComponent/NativeComponent.vcxproj.filters @@ -6,4 +6,12 @@ rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx;tiff;tif;png;wav;mfcribbon-ms - + + + + + + + + + \ No newline at end of file diff --git a/NativeComponent/pch.h b/NativeComponent/pch.h index f815ac9..c27fe20 100644 --- a/NativeComponent/pch.h +++ b/NativeComponent/pch.h @@ -4,3 +4,7 @@ // #pragma once +#include +#include +#include +#include "NativeComponent.h" \ No newline at end of file diff --git a/NativeFilterDemo/App.xaml b/NativeFilterDemo/App.xaml index 55d7bd3..32ed54f 100644 --- a/NativeFilterDemo/App.xaml +++ b/NativeFilterDemo/App.xaml @@ -2,7 +2,6 @@ x:Class="NativeFilterDemo.App" xmlns="http://schemas.microsoft.com/winfx/2006/xaml/presentation" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" - xmlns:phone="clr-namespace:Microsoft.Phone.Controls;assembly=Microsoft.Phone" xmlns:shell="clr-namespace:Microsoft.Phone.Shell;assembly=Microsoft.Phone"> diff --git a/NativeFilterDemo/App.xaml.cs b/NativeFilterDemo/App.xaml.cs index 977c4b4..2c9d6df 100644 --- a/NativeFilterDemo/App.xaml.cs +++ b/NativeFilterDemo/App.xaml.cs @@ -1,6 +1,5 @@ using System; using System.Diagnostics; -using System.Resources; using System.Windows; using System.Windows.Markup; using System.Windows.Navigation; diff --git a/NativeFilterDemo/AspectRatioValueConverter.cs b/NativeFilterDemo/AspectRatioValueConverter.cs index a2de9af..f85da31 100644 --- a/NativeFilterDemo/AspectRatioValueConverter.cs +++ b/NativeFilterDemo/AspectRatioValueConverter.cs @@ -1,9 +1,5 @@ using System; -using System.Collections.Generic; using System.Globalization; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using System.Windows.Data; namespace NativeFilterDemo diff --git a/NativeFilterDemo/Assets/Icons/back.png b/NativeFilterDemo/Assets/Icons/back.png new file mode 100644 index 0000000..c0119df Binary files /dev/null and b/NativeFilterDemo/Assets/Icons/back.png differ diff --git a/NativeFilterDemo/Assets/Icons/next.png b/NativeFilterDemo/Assets/Icons/next.png new file mode 100644 index 0000000..c789c4e Binary files /dev/null and b/NativeFilterDemo/Assets/Icons/next.png differ diff --git a/NativeFilterDemo/CameraStreamSource.cs b/NativeFilterDemo/CameraStreamSource.cs index d54e835..fd49d63 100644 --- a/NativeFilterDemo/CameraStreamSource.cs +++ b/NativeFilterDemo/CameraStreamSource.cs @@ -1,45 +1,38 @@ -using NativeComponent; -using System; -using System.Collections.Generic; -using System.Globalization; -using System.IO; -using System.Windows.Media; - -namespace NativeFilterDemo +namespace NativeFilterDemo { + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Globalization; + using System.IO; + using System.Runtime.InteropServices.WindowsRuntime; + using System.Threading.Tasks; + using System.Windows.Media; + using System.Windows.Threading; + using Windows.Foundation; + using CameraEffectInterface; + + /// + /// A source for the media element. Feeds the Media Element with frames coming from the + /// ICameraEffect implementation. + /// public class CameraStreamSource : MediaStreamSource { - - private MediaStreamDescription _videoStreamDescription; - - private int _frameTime = 0; - private long _currentTime = 0; - private int _frameWidth; - private int _frameHeight; - private const int _framePixelSize = 4; // RGBA - private int _frameBufferSize; - private int _frameStreamSize; - - private MemoryStream _frameStream; - private int _frameStreamOffset = 0; - private Dictionary _emptySampleDict = + private readonly Dictionary emptySampleDict = new Dictionary(); - WindowsPhoneRuntimeComponent _cameraBuffer; - int[] _cameraData; - byte[] _cameraFilteredData; + private long currentTime; + private int frameStreamOffset; + private int frameTime; + private MediaStreamDescription videoStreamDescription; - public CameraStreamSource(WindowsPhoneRuntimeComponent cameraBuffer, Windows.Foundation.Size size) + public CameraStreamSource(ICameraEffect cameraEffect, Size targetMediaElementSize) { - _cameraBuffer = cameraBuffer; - _frameWidth = (int)size.Width; - _frameHeight = (int)size.Height; - - _cameraData = new int[_frameWidth * _frameHeight]; - _frameBufferSize = _frameWidth * _frameHeight * _framePixelSize; - _cameraFilteredData = new byte[_frameBufferSize]; - _frameStreamSize = _frameBufferSize * 4; //Number of frames for buffering : 4 works well. - _frameStream = new MemoryStream(_frameStreamSize); + CameraStreamSourceDataSingleton dataSource = CameraStreamSourceDataSingleton.Instance; + dataSource.Initialize(targetMediaElementSize); + dataSource.CameraEffect = cameraEffect; + cameraEffect.OutputBufferSize = targetMediaElementSize; + cameraEffect.OutputBuffer = dataSource.ImageBuffer.AsBuffer(); } protected override void OpenMediaAsync() @@ -49,12 +42,14 @@ protected override void OpenMediaAsync() Dictionary mediaStreamAttributes = new Dictionary(); List mediaStreamDescriptions = new List(); + CameraStreamSourceDataSingleton dataSource = CameraStreamSourceDataSingleton.Instance; + mediaStreamAttributes[MediaStreamAttributeKeys.VideoFourCC] = "RGBA"; - mediaStreamAttributes[MediaStreamAttributeKeys.Width] = _frameWidth.ToString(); - mediaStreamAttributes[MediaStreamAttributeKeys.Height] = _frameHeight.ToString(); + mediaStreamAttributes[MediaStreamAttributeKeys.Width] = dataSource.FrameWidth.ToString(); + mediaStreamAttributes[MediaStreamAttributeKeys.Height] = dataSource.FrameHeight.ToString(); - _videoStreamDescription = new MediaStreamDescription(MediaStreamType.Video, mediaStreamAttributes); - mediaStreamDescriptions.Add(_videoStreamDescription); + videoStreamDescription = new MediaStreamDescription(MediaStreamType.Video, mediaStreamAttributes); + mediaStreamDescriptions.Add(videoStreamDescription); // a zero timespan is an infinite video mediaSourceAttributes[MediaSourceAttributesKeys.Duration] = @@ -62,37 +57,44 @@ protected override void OpenMediaAsync() mediaSourceAttributes[MediaSourceAttributesKeys.CanSeek] = false.ToString(); - _frameTime = (int)TimeSpan.FromSeconds((double)1 / 30).Ticks; + frameTime = (int)TimeSpan.FromSeconds((double)0).Ticks; + // Report that we finished initializing its internal state and can now // pass in frame samples. - ReportOpenMediaCompleted(mediaSourceAttributes, mediaStreamDescriptions); } - protected override void GetSampleAsync(MediaStreamType mediaStreamType) { - System.Diagnostics.Debug.WriteLine("GetSampleAsync in"); - if (_frameStreamOffset + _frameBufferSize > _frameStreamSize) + CameraStreamSourceDataSingleton dataSource = CameraStreamSourceDataSingleton.Instance; + + if (frameStreamOffset + dataSource.FrameBufferSize > dataSource.FrameStreamSize) { - _frameStream.Seek(0, SeekOrigin.Begin); - _frameStreamOffset = 0; + dataSource.FrameStream.Seek(0, SeekOrigin.Begin); + frameStreamOffset = 0; } - _cameraBuffer.NewViewfinderFrame(_cameraData, _cameraFilteredData); - _frameStream.Write(_cameraFilteredData, 0, _frameBufferSize); - - MediaStreamSample msSamp = new MediaStreamSample( - _videoStreamDescription, _frameStream, _frameStreamOffset, - _frameBufferSize, _currentTime, _emptySampleDict); - - ReportGetSampleCompleted(msSamp); - - _currentTime += _frameTime; - _frameStreamOffset += _frameBufferSize; - - System.Diagnostics.Debug.WriteLine("GetSampleAsync out"); + Task tsk = dataSource.CameraEffect.GetNewFrameAndApplyEffect().AsTask(); + + // Wait that the asynchroneous call completes, and proceed by reporting + // the MediaElement that new samples are ready. + tsk.ContinueWith((task) => + { + dataSource.FrameStream.Position = 0; + + MediaStreamSample msSamp = new MediaStreamSample( + videoStreamDescription, + dataSource.FrameStream, + frameStreamOffset, + dataSource.FrameBufferSize, + currentTime, + emptySampleDict); + + ReportGetSampleCompleted(msSamp); + currentTime += frameTime; + frameStreamOffset += dataSource.FrameBufferSize; + }); } protected override void CloseMedia() @@ -106,8 +108,8 @@ protected override void GetDiagnosticAsync(MediaStreamSourceDiagnosticKind diagn protected override void SeekAsync(long seekToTime) { - _currentTime = seekToTime; - this.ReportSeekCompleted(seekToTime); + currentTime = seekToTime; + ReportSeekCompleted(seekToTime); } protected override void SwitchMediaStreamAsync(MediaStreamDescription mediaStreamDescription) @@ -115,6 +117,5 @@ protected override void SwitchMediaStreamAsync(MediaStreamDescription mediaStrea throw new NotImplementedException(); } - } -} +} \ No newline at end of file diff --git a/NativeFilterDemo/CameraStreamSourceDataSingleton.cs b/NativeFilterDemo/CameraStreamSourceDataSingleton.cs new file mode 100644 index 0000000..aff4b92 --- /dev/null +++ b/NativeFilterDemo/CameraStreamSourceDataSingleton.cs @@ -0,0 +1,72 @@ +namespace NativeFilterDemo +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Text; + using System.Threading.Tasks; + using System.Windows; + using CameraEffectInterface; + + /// + /// A class to work around a problem with MediaElement. + /// MediaElement doesn't destruct the current MediaStreamSource when the source is changed. + /// To avoid memory leaks when source is changed (for example when navigating away from a page) + /// reserve the memory once and use it for all the MediaStreamSource instances we create. + /// + public sealed class CameraStreamSourceDataSingleton + { + private const int FramePixelSize = 4; // RGBA + private const int NumberOfBufferedFrames = 1; + + private static CameraStreamSourceDataSingleton classInstance = null; + public static CameraStreamSourceDataSingleton Instance + { + get + { + if (classInstance == null) + { + classInstance = new CameraStreamSourceDataSingleton(); + } + + return classInstance; + } + } + + public byte[] ImageBuffer { get; private set; } + public MemoryStream FrameStream { get; private set; } + public int FrameHeight { get; private set; } + public int FrameWidth { get; private set; } + public int FrameBufferSize { get; private set; } + public int FrameStreamSize { get; private set; } + public ICameraEffect CameraEffect { get; set; } + + /// + /// Create the buffers of the data source + /// + /// The dimensions of a camera frame + public void Initialize(Windows.Foundation.Size cameraFrameSize) + { + if (this.FrameWidth != 0) + { + if (((int)cameraFrameSize.Width != this.FrameWidth) + || ((int)cameraFrameSize.Height != this.FrameHeight)) + { + // Currently, we don't allow changing the frame size + // after first initialization + throw new NotSupportedException(); + } + + return; // singleton is already initialized. + } + + this.FrameWidth = (int)cameraFrameSize.Width; + this.FrameHeight = (int)cameraFrameSize.Height; + this.FrameBufferSize = FrameWidth * FrameHeight * FramePixelSize; + this.ImageBuffer = new byte[FrameBufferSize]; + this.FrameStreamSize = FrameBufferSize * NumberOfBufferedFrames; + this.FrameStream = new MemoryStream(ImageBuffer); + } + } +} diff --git a/NativeFilterDemo/MainPage.xaml b/NativeFilterDemo/MainPage.xaml index 50fcec0..85c90e4 100644 --- a/NativeFilterDemo/MainPage.xaml +++ b/NativeFilterDemo/MainPage.xaml @@ -6,76 +6,31 @@ xmlns:shell="clr-namespace:Microsoft.Phone.Shell;assembly=Microsoft.Phone" xmlns:d="http://schemas.microsoft.com/expression/blend/2008" xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006" - xmlns:local="clr-namespace:NativeFilterDemo" mc:Ignorable="d" FontFamily="{StaticResource PhoneFontFamilyNormal}" FontSize="{StaticResource PhoneFontSizeNormal}" Foreground="{StaticResource PhoneForegroundBrush}" - SupportedOrientations="Landscape" Orientation="Landscape" + SupportedOrientations="Landscape" Orientation="LandscapeLeft" shell:SystemTray.IsVisible="False"> - - - - - + - - - - - - - - - - - - - - - - - + + + + + Stretch="UniformToFill" + BufferingTime="0" + Tap="MyCameraMediaElement_Tap"/> + - - - - - + \ No newline at end of file diff --git a/NativeFilterDemo/MainPage.xaml.cs b/NativeFilterDemo/MainPage.xaml.cs index 8708933..51315dd 100644 --- a/NativeFilterDemo/MainPage.xaml.cs +++ b/NativeFilterDemo/MainPage.xaml.cs @@ -1,88 +1,98 @@ -using System; -using System.Collections.Generic; + +using System; using System.Linq; -using System.Net; -using System.Windows; -using System.Windows.Media; -using System.Windows.Controls; using System.Windows.Navigation; +using System.Windows.Threading; using Microsoft.Phone.Controls; using Microsoft.Phone.Shell; -using NativeFilterDemo.Resources; -using NativeComponent; -using Microsoft.Devices; using Windows.Phone.Media.Capture; -using System.Windows.Media.Imaging; -using System.Windows.Threading; -using System.Threading; -using Microsoft.Phone; - +using Size = Windows.Foundation.Size; +using System.Diagnostics; +using CameraEffectInterface; +using NativeComponent; namespace NativeFilterDemo { public partial class MainPage : PhoneApplicationPage { - PhotoCaptureDevice m_camera; - WindowsPhoneRuntimeComponent m_nativeFilter; - - DateTime m_startTime; - DispatcherTimer m_timer; + private PhotoCaptureDevice m_camera; + private readonly ICameraEffect cameraEffect; + private CameraStreamSource source; + private DispatcherTimer m_timer; // Constructor public MainPage() { InitializeComponent(); - Loaded += MainPage_Loaded; + cameraEffect = new WindowsPhoneRuntimeComponent(); + BuildApplicationBar(); } - async void MainPage_Loaded(object sender, RoutedEventArgs e) + private void BuildApplicationBar() { - Microsoft.Phone.Shell.PhoneApplicationService.Current.ApplicationIdleDetectionMode = IdleDetectionMode.Disabled; - Windows.Foundation.Size resolution = new Windows.Foundation.Size(640, 480); - m_camera = await PhotoCaptureDevice.OpenAsync(CameraSensorLocation.Back, resolution); + ApplicationBar = new ApplicationBar(); - Windows.Foundation.Size actualResolution = m_camera.PreviewResolution; - - // The viewfinderbrush (preview of the original data from camera) - // is currently very unstable on the device, when used together - // with a MediaEngine playback component. - - //ViewfinderBrush.SetSource(m_camera); + ApplicationBarIconButton button = new ApplicationBarIconButton(new Uri("/Assets/Icons/next.png", UriKind.Relative)); + button.Text = "Second page"; + button.Click += SecondPageButtonClick; + ApplicationBar.Buttons.Add(button); + Loaded += MainPage_Loaded; + } - m_nativeFilter = new WindowsPhoneRuntimeComponent(); - m_nativeFilter.Initialize(m_camera); + async void MainPage_Loaded(object sender, System.Windows.RoutedEventArgs e) + { + Size targetMediaElementSize = new Size(640, 480); + double aspectRatio = 4.0/3.0; - CameraStreamSource source = new CameraStreamSource(m_nativeFilter, actualResolution); - MyCameraMediaElement.SetSource (source); + // 1. Open camera + if (m_camera == null) + { + var captureRes = PhotoCaptureDevice.GetAvailableCaptureResolutions(CameraSensorLocation.Back); + Size selectedCaptureRes = captureRes.Where(res => Math.Abs(aspectRatio - res.Width/res.Height ) <= 0.1) + .OrderBy(res => res.Width) + .Last(); + m_camera = await PhotoCaptureDevice.OpenAsync(CameraSensorLocation.Back, selectedCaptureRes); + m_camera.SetProperty(KnownCameraGeneralProperties.EncodeWithOrientation, m_camera.SensorLocation == CameraSensorLocation.Back ? m_camera.SensorRotationInDegrees : -m_camera.SensorRotationInDegrees); + + var previewRes = PhotoCaptureDevice.GetAvailablePreviewResolutions(CameraSensorLocation.Back); + Size selectedPreviewRes = previewRes.Where(res => Math.Abs(aspectRatio - res.Width/res.Height ) <= 0.1) + .Where(res => (res.Height >= targetMediaElementSize.Height) && (res.Width >= targetMediaElementSize.Width)) + .OrderBy(res => res.Width) + .First(); + await m_camera.SetPreviewResolutionAsync(selectedPreviewRes); + cameraEffect.CaptureDevice = m_camera; + } + + // Always create a new source, otherwise the MediaElement will not start. + source = new CameraStreamSource(cameraEffect, targetMediaElementSize); + MyCameraMediaElement.SetSource(source); - m_startTime = DateTime.Now; m_timer = new DispatcherTimer(); m_timer.Interval = new TimeSpan(0, 0, 0, 1, 0); // Tick every 1s. m_timer.Tick += m_timer_Tick; m_timer.Start(); - } - void m_timer_Tick(object sender, EventArgs e) { - System.Diagnostics.Debug.WriteLine("FPS:" + MyCameraMediaElement.RenderedFramesPerSecond ); + System.Diagnostics.Debug.WriteLine("FPS 2:" + MyCameraMediaElement.RenderedFramesPerSecond); } - - - protected override void OnNavigatedTo(NavigationEventArgs e) + private void SecondPageButtonClick(object sender, EventArgs e) { - if (m_camera != null) - { - ViewfinderBrush.SetSource(m_camera); - } + if (m_camera == null) return; + + MyCameraMediaElement.Source = null; + NavigationService.Navigate(new Uri("/SecondPage.xaml", UriKind.Relative)); + } - protected override void OnNavigatedFrom(NavigationEventArgs e) + private void MyCameraMediaElement_Tap(object sender, System.Windows.Input.GestureEventArgs e) { - System.Diagnostics.Debug.WriteLine("OnNavigatedFrom"); - base.OnNavigatedFrom(e); + if (cameraEffect != null) + { + cameraEffect.ChangeEffectType(); + } } } } \ No newline at end of file diff --git a/NativeFilterDemo/NativeFilterDemo.csproj b/NativeFilterDemo/NativeFilterDemo.csproj index 02741e8..316536d 100644 --- a/NativeFilterDemo/NativeFilterDemo.csproj +++ b/NativeFilterDemo/NativeFilterDemo.csproj @@ -90,6 +90,7 @@ 4 + App.xaml @@ -105,6 +106,9 @@ True AppResources.resx + + SecondPage.xaml + @@ -115,6 +119,10 @@ Designer MSBuild:Compile + + Designer + MSBuild:Compile + @@ -129,6 +137,8 @@ PreserveNewest + + PreserveNewest @@ -153,6 +163,10 @@ + + {B54D292C-6154-413F-9C4C-D4A08CDB7EDF} + ICameraEffect + {ECCD1443-9EAA-4666-BC16-6AA5B5C11BA2} NativeComponent diff --git a/NativeFilterDemo/SecondPage.xaml b/NativeFilterDemo/SecondPage.xaml new file mode 100644 index 0000000..0b9c957 --- /dev/null +++ b/NativeFilterDemo/SecondPage.xaml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/NativeFilterDemo/SecondPage.xaml.cs b/NativeFilterDemo/SecondPage.xaml.cs new file mode 100644 index 0000000..34800e0 --- /dev/null +++ b/NativeFilterDemo/SecondPage.xaml.cs @@ -0,0 +1,30 @@ +using System; +using Microsoft.Phone.Controls; +using Microsoft.Phone.Shell; + +namespace NativeFilterDemo +{ + public partial class SecondPage : PhoneApplicationPage + { + public SecondPage() + { + InitializeComponent(); + BuildApplicationBar(); + } + + private void BuildApplicationBar() + { + ApplicationBar = new ApplicationBar(); + + ApplicationBarIconButton button = new ApplicationBarIconButton(new Uri("/Assets/Icons/back.png", UriKind.Relative)); + button.Text = "Camera page"; + button.Click += BackButtonClick; + ApplicationBar.Buttons.Add(button); + } + + private void BackButtonClick(object sender, EventArgs e) + { + NavigationService.GoBack(); + } + } +} \ No newline at end of file