Skip to content

Commit

Permalink
Windows 10 Anniversary Update - March 2017 Update 2
Browse files Browse the repository at this point in the history
  • Loading branch information
@oldnewthing
oldnewthing committed Mar 9, 2017
2 parents c29f596 + afe2b4a commit 3892c2d
Show file tree
Hide file tree
Showing 117 changed files with 4,820 additions and 596 deletions.
117 changes: 83 additions & 34 deletions Samples/CameraFrames/cpp/FrameRenderer.cpp
View file
Expand Up @@ -22,6 +22,7 @@ using namespace Microsoft::WRL;
using namespace Windows::Foundation;
using namespace Windows::Graphics::Imaging;
using namespace Windows::Media::Capture::Frames;
using namespace Windows::Media::MediaProperties;
using namespace Windows::UI::Xaml::Controls;
using namespace Windows::UI::Xaml::Media::Imaging;

Expand Down Expand Up @@ -236,6 +237,33 @@ void FrameRenderer::ProcessFrame(Windows::Media::Capture::Frames::MediaFrameRefe
}
}

String^ FrameRenderer::GetSubtypeForFrameReader(MediaFrameSourceKind kind, MediaFrameFormat^ format)
{
// Note that media encoding subtypes may differ in case.
// https://docs.microsoft.com/en-us/uwp/api/Windows.Media.MediaProperties.MediaEncodingSubtypes

String^ subtype = format->Subtype;
switch (kind)
{
// For color sources, we accept anything and request that it be converted to Bgra8.
case MediaFrameSourceKind::Color:
return MediaEncodingSubtypes::Bgra8;

// The only depth format we can render is D16.
case MediaFrameSourceKind::Depth:
return CompareStringOrdinal(subtype->Data(), -1, L"D16", -1, TRUE) == CSTR_EQUAL ? subtype : nullptr;

// The only infrared formats we can render are L8 and L16.
case MediaFrameSourceKind::Infrared:
return (CompareStringOrdinal(subtype->Data(), -1, L"L8", -1, TRUE) == CSTR_EQUAL ||
CompareStringOrdinal(subtype->Data(), -1, L"L16", -1, TRUE) == CSTR_EQUAL) ? subtype : nullptr;

// No other source kinds are supported by this class.
default:
return nullptr;
}
}

SoftwareBitmap^ FrameRenderer::ConvertToDisplayableImage(VideoMediaFrame^ inputFrame)
{
if (inputFrame == nullptr)
Expand All @@ -244,45 +272,66 @@ SoftwareBitmap^ FrameRenderer::ConvertToDisplayableImage(VideoMediaFrame^ inputF
}

SoftwareBitmap^ inputBitmap = inputFrame->SoftwareBitmap;
if ((inputBitmap->BitmapPixelFormat == BitmapPixelFormat::Bgra8) &&
(inputBitmap->BitmapAlphaMode == BitmapAlphaMode::Premultiplied))
{
// SoftwareBitmap is already in the correct format for an Image control, so just return a copy.
return SoftwareBitmap::Copy(inputBitmap);
}
else if ((inputBitmap->BitmapPixelFormat == BitmapPixelFormat::Gray16) && (inputFrame->FrameReference->SourceKind == MediaFrameSourceKind::Depth))
{
using namespace std::placeholders;

// Use a special pseudo color to render 16 bits depth frame.
// Since we must scale the output appropriately we use std::bind to
// create a function that takes the depth scale as input but also matches
// the required signature.
double depthScale = inputFrame->DepthMediaFrame->DepthFormat->DepthScaleInMeters;
return TransformBitmap(inputBitmap, std::bind(&PseudoColorForDepth, _1, _2, _3, static_cast<float>(depthScale)));
}
else if (inputBitmap->BitmapPixelFormat == BitmapPixelFormat::Gray16)
{
// Use pseudo color to render 16 bits frames.
return TransformBitmap(inputBitmap, PseudoColorFor16BitInfrared);
}
else if (inputBitmap->BitmapPixelFormat == BitmapPixelFormat::Gray8)
{
// Use pseudo color to render 8 bits frames.
return TransformBitmap(inputBitmap, PseudoColorFor8BitInfrared);
}
else
auto mode = inputBitmap->BitmapAlphaMode;

switch (inputFrame->FrameReference->SourceKind)
{
try
case MediaFrameSourceKind::Color:
// XAML requires Bgra8 with premultiplied alpha.
// We requested Bgra8 from the MediaFrameReader, so all that's
// left is fixing the alpha channel if necessary.
if (inputBitmap->BitmapPixelFormat != BitmapPixelFormat::Bgra8)
{
OutputDebugStringW(L"Color format should have been Bgra8.\r\n");
}
else if (inputBitmap->BitmapAlphaMode == BitmapAlphaMode::Premultiplied)
{
// Convert to Bgra8 Premultiplied SoftwareBitmap, so xaml can display in UI.
// Already in the correct format.
return SoftwareBitmap::Copy(inputBitmap);
}
else
{
// Convert to premultiplied alpha.
return SoftwareBitmap::Convert(inputBitmap, BitmapPixelFormat::Bgra8, BitmapAlphaMode::Premultiplied);
}
catch (InvalidArgumentException^ exception)
return nullptr;

case MediaFrameSourceKind::Depth:
// We requested D16 from the MediaFrameReader, so the frame should
// be in Gray16 format.

if (inputBitmap->BitmapPixelFormat == BitmapPixelFormat::Gray16)
{
using namespace std::placeholders;

// Use a special pseudo color to render 16 bits depth frame.
// Since we must scale the output appropriately we use std::bind to
// create a function that takes the depth scale as input but also matches
// the required signature.
double depthScale = inputFrame->DepthMediaFrame->DepthFormat->DepthScaleInMeters;
return TransformBitmap(inputBitmap, std::bind(&PseudoColorForDepth, _1, _2, _3, static_cast<float>(depthScale)));
}
else
{
// Conversion of software bitmap format is not supported. Drop this frame.
OutputDebugStringW(exception->Message->Data());
OutputDebugStringW(L"\r\n");
OutputDebugStringW(L"Depth format in unexpected format.\r\n");
}
return nullptr;

case MediaFrameSourceKind::Infrared:
// We requested L8 or L16 from the MediaFrameReader, so the frame should
// be in Gray8 or Gray16 format.
switch (inputBitmap->BitmapPixelFormat)
{
case BitmapPixelFormat::Gray8:
// Use pseudo color to render 8 bits frames.
return TransformBitmap(inputBitmap, PseudoColorFor8BitInfrared);

case BitmapPixelFormat::Gray16:
// Use pseudo color to render 16 bits frames.
return TransformBitmap(inputBitmap, PseudoColorFor16BitInfrared);

default:
OutputDebugStringW(L"Infrared format should have been Gray8 or Gray16.\r\n");
return nullptr;
}
}
Expand Down
11 changes: 10 additions & 1 deletion Samples/CameraFrames/cpp/FrameRenderer.h
View file
Expand Up @@ -29,14 +29,23 @@ namespace SDKTemplate
/// </summary>
void ProcessFrame(Windows::Media::Capture::Frames::MediaFrameReference^ frame);

/// <summary>
/// Determines the subtype to request from the MediaFrameReader that will result in
/// a frame that can be rendered by ConvertToDisplayableImage.
/// </summary>
/// <returns>Subtype string to request, or null if subtype is not renderable.</returns>
static Platform::String^ GetSubtypeForFrameReader(
Windows::Media::Capture::Frames::MediaFrameSourceKind kind,
Windows::Media::Capture::Frames::MediaFrameFormat^ format);

private: // Private static methods.
/// <summary>
/// Converts the input frame to BGRA8 premultiplied alpha format and returns the result.
/// Returns nullptr if the input frame cannot be converted BGRA8 premultiplied alpha.
/// </summary>
static Windows::Graphics::Imaging::SoftwareBitmap^ ConvertToDisplayableImage(
Windows::Media::Capture::Frames::VideoMediaFrame^ inputFrame);

/// <summary>
/// Transforms pixels of inputBitmap to an output bitmap using the supplied pixel transformation method.
/// Returns nullptr if translation fails.
Expand Down
165 changes: 76 additions & 89 deletions Samples/CameraFrames/cpp/Scenario1_DisplayDepthColorIR.xaml.cpp
View file
Expand Up @@ -12,6 +12,7 @@
#include "pch.h"
#include "Scenario1_DisplayDepthColorIR.xaml.h"
#include "FrameRenderer.h"
#include <unordered_set>

using namespace SDKTemplate;

Expand All @@ -25,16 +26,6 @@ using namespace Windows::Media::Capture;
using namespace Windows::Media::Capture::Frames;
using namespace Windows::UI::Xaml::Media::Imaging;

MediaFrameSourceInfo^ Scenario1_DisplayDepthColorIR::GetFirstSourceInfoOfKind(MediaFrameSourceGroup^ group, MediaFrameSourceKind kind)
{
auto matchingInfo = std::find_if(begin(group->SourceInfos), end(group->SourceInfos),
[kind](MediaFrameSourceInfo^ sourceInfo)
{
return sourceInfo->SourceKind == kind;
});
return (matchingInfo != end(group->SourceInfos)) ? *matchingInfo : nullptr;
}

Scenario1_DisplayDepthColorIR::Scenario1_DisplayDepthColorIR()
{
InitializeComponent();
Expand Down Expand Up @@ -73,114 +64,110 @@ task<void> Scenario1_DisplayDepthColorIR::PickNextMediaSourceAsync()

task<void> Scenario1_DisplayDepthColorIR::PickNextMediaSourceWorkerAsync()
{
struct EligibleGroup
{
MediaFrameSourceGroup^ Group;
std::array<MediaFrameSourceInfo^, 3> SourceInfos;
};

return CleanupMediaCaptureAsync()
.then([this]()
{
return create_task(MediaFrameSourceGroup::FindAllAsync());
}).then([this](IVectorView<MediaFrameSourceGroup^>^ allGroups)
}, task_continuation_context::get_current_winrt_context())
.then([this](IVectorView<MediaFrameSourceGroup^>^ allGroups)
{
// Identify the color, depth, and infrared sources of each group,
// and keep only the groups that have at least one recognized source.
std::vector<EligibleGroup> eligibleGroups;
for (auto group : allGroups)
{
EligibleGroup eligibleGroup;
eligibleGroup.Group = group;
// For each source kind, find the source which offers that kind of media frame,
// or null if there is no such source.
eligibleGroup.SourceInfos[0] = GetFirstSourceInfoOfKind(group, MediaFrameSourceKind::Color);
eligibleGroup.SourceInfos[1] = GetFirstSourceInfoOfKind(group, MediaFrameSourceKind::Depth);
eligibleGroup.SourceInfos[2] = GetFirstSourceInfoOfKind(group, MediaFrameSourceKind::Infrared);
// If any source was found of any kind we support, keep this group.
if (std::any_of(eligibleGroup.SourceInfos.begin(), eligibleGroup.SourceInfos.end(),
[](MediaFrameSourceInfo^ sourceInfo) { return sourceInfo != nullptr; }))
{
eligibleGroups.push_back(eligibleGroup);
}
}

if (eligibleGroups.size() == 0)
if (allGroups->Size == 0)
{
m_logger->Log("No source group with color, depth or infrared found.");
m_logger->Log("No source groups found.");
return task_from_result();
}

// Pick next group in the array after each time the Next button is clicked.
m_selectedSourceGroupIndex = (m_selectedSourceGroupIndex + 1) % eligibleGroups.size();
m_selectedSourceGroupIndex = (m_selectedSourceGroupIndex + 1) % allGroups->Size;
MediaFrameSourceGroup^ selectedGroup = allGroups->GetAt(m_selectedSourceGroupIndex);

m_logger->Log("Found " + eligibleGroups.size().ToString() + " groups and " +
m_logger->Log("Found " + allGroups->Size.ToString() + " groups and " +
"selecting index [" + m_selectedSourceGroupIndex.ToString() + "] : " +
eligibleGroups[m_selectedSourceGroupIndex].Group->DisplayName);
EligibleGroup selected = eligibleGroups[m_selectedSourceGroupIndex];
selectedGroup->DisplayName);

// Initialize MediaCapture with selected group.
return TryInitializeMediaCaptureAsync(selected.Group)
.then([this, selected](bool initialized)
return TryInitializeMediaCaptureAsync(selectedGroup)
.then([this, selectedGroup](bool initialized)
{
if (!initialized)
{
return CleanupMediaCaptureAsync();
}

// Set up frame readers, register event handlers and start streaming.
auto startedKinds = std::make_shared<std::unordered_set<MediaFrameSourceKind>>();
task<void> createReadersTask = task_from_result();
for (size_t i = 0; i < selected.SourceInfos.size(); i++)
for (IKeyValuePair<String^, MediaFrameSource^>^ kvp : m_mediaCapture->FrameSources)
{
MediaFrameSourceInfo^ info = selected.SourceInfos[i];
if (info != nullptr)
MediaFrameSource^ source = kvp->Value;
createReadersTask = createReadersTask.then([this, startedKinds, source]()
{
createReadersTask = createReadersTask && CreateReaderAsync(selected.Group, info);
}
else
MediaFrameSourceKind kind = source->Info->SourceKind;

// Ignore this source if we already have a source of this kind.
if (startedKinds->find(kind) != startedKinds->end())
{
return task_from_result();
}

// Look for a format which the FrameRenderer can render.
String^ requestedSubtype = nullptr;
auto found = std::find_if(begin(source->SupportedFormats), end(source->SupportedFormats),
[&](MediaFrameFormat^ format)
{
requestedSubtype = FrameRenderer::GetSubtypeForFrameReader(kind, format);
return requestedSubtype != nullptr;
});
if (requestedSubtype == nullptr)
{
// No acceptable format was found. Ignore this source.
return task_from_result();
}

// Tell the source to use the format we can render.
return create_task(source->SetFormatAsync(*found))
.then([this, source, requestedSubtype]()
{
return create_task(m_mediaCapture->CreateFrameReaderAsync(source, requestedSubtype));
}, task_continuation_context::get_current_winrt_context())
.then([this, kind](MediaFrameReader^ frameReader)
{
EventRegistrationToken token = frameReader->FrameArrived +=
ref new TypedEventHandler<MediaFrameReader^, MediaFrameArrivedEventArgs^>(this, &Scenario1_DisplayDepthColorIR::FrameReader_FrameArrived);

// Keep track of created reader and event handler so it can be stopped later.
m_readers.push_back(std::make_pair(frameReader, token));

m_logger->Log(kind.ToString() + " reader created");

return create_task(frameReader->StartAsync());
}, task_continuation_context::get_current_winrt_context())
.then([this, kind, startedKinds](MediaFrameReaderStartStatus status)
{
if (status == MediaFrameReaderStartStatus::Success)
{
m_logger->Log("Started " + kind.ToString() + " reader.");
startedKinds->insert(kind);
}
else
{
m_logger->Log("Unable to start " + kind.ToString() + " reader. Error: " + status.ToString());
}
}, task_continuation_context::get_current_winrt_context());
}, task_continuation_context::get_current_winrt_context());
}
// Run the loop and see if any sources were used.
return createReadersTask.then([this, startedKinds, selectedGroup]()
{
if (startedKinds->size() == 0)
{
String^ frameKind = (i == 0 ? "Color" : i == 1 ? "Depth" : "Infrared");
m_logger->Log("No " + frameKind + " source in group " + selected.Group->DisplayName);
m_logger->Log("No eligible sources in " + selectedGroup->DisplayName + ".");
}

}
return createReadersTask;
});
}, task_continuation_context::get_current_winrt_context());
}, task_continuation_context::get_current_winrt_context());
}, task_continuation_context::get_current_winrt_context());
}

task<void> Scenario1_DisplayDepthColorIR::CreateReaderAsync(MediaFrameSourceGroup^ group, MediaFrameSourceInfo^ info)
{
// Access the initialized frame source by looking up the the Id of the source.
// Verify that the Id is present, because it may have left the group while were were
// busy deciding which group to use.
if (!m_mediaCapture->FrameSources->HasKey(info->Id))
{
m_logger->Log("Unable to start " + info->SourceKind.ToString() + " reader: Frame source not found");
return task_from_result();
}

return create_task(m_mediaCapture->CreateFrameReaderAsync(m_mediaCapture->FrameSources->Lookup(info->Id)))
.then([this, info](MediaFrameReader^ frameReader)
{
EventRegistrationToken token = frameReader->FrameArrived +=
ref new TypedEventHandler<MediaFrameReader^, MediaFrameArrivedEventArgs^>(this, &Scenario1_DisplayDepthColorIR::FrameReader_FrameArrived);

m_logger->Log(info->SourceKind.ToString() + " reader created");

// Keep track of created reader and event handler so it can be stopped later.
m_readers.push_back(std::make_pair(frameReader, token));

return create_task(frameReader->StartAsync());
}).then([this, info](MediaFrameReaderStartStatus status)
{
if (status != MediaFrameReaderStartStatus::Success)
{
m_logger->Log("Unable to start " + info->SourceKind.ToString() + " reader. Error: " + status.ToString());
}
});
}

task<bool> Scenario1_DisplayDepthColorIR::TryInitializeMediaCaptureAsync(MediaFrameSourceGroup^ group)
{
if (m_mediaCapture != nullptr)
Expand Down

0 comments on commit 3892c2d

Please sign in to comment.