[Fabric] Support blob: images, and provide better image.onError callb…

…acks (microsoft#13285)

* [Fabric] Support blob: images, and provide better image.onError callbacks

* Change files

* Remove some test code

* Add user-agent

* Add public API to set default User-Agent header

* various fixes

* fix snapshots

* Rename Networking -> HttpSettings

* Change files

* fix

* Http.UserAgent RuntimeOption needs to be set before the HttpResource is created

change/react-native-windows-c33514c0-2412-40ff-b4ad-4739c95260e7.json

@@ -0,0 +1,7 @@
+{
+  "type": "patch",
+  "comment": "[Fabric] Support blob: images, and provide better image.onError callbacks",
+  "packageName": "react-native-windows",
+  "email": "30809111+acoates-ms@users.noreply.github.com",
+  "dependentChangeType": "patch"
+}

packages/playground/windows/playground-composition/Playground-Composition.cpp

@@ -52,43 +52,41 @@ void RegisterCustomComponent(winrt::Microsoft::ReactNative::IReactPackageBuilder
  *
  * This allows applications to provide custom image rendering pipelines.
  */
-struct EllipseImageHandler : winrt::implements<
-                                 EllipseImageHandler,
-                                 winrt::Microsoft::ReactNative::Composition::Experimental::IUriBrushProvider,
-                                 winrt::Microsoft::ReactNative::Composition::IUriImageProvider> {
+struct EllipseImageHandler
+    : winrt::implements<EllipseImageHandler, winrt::Microsoft::ReactNative::Composition::IUriImageProvider> {
   bool CanLoadImageUri(winrt::Microsoft::ReactNative::IReactContext context, winrt::Windows::Foundation::Uri uri) {
     return uri.SchemeName() == L"ellipse";
   }
 
-  winrt::Windows::Foundation::IAsyncOperation<winrt::Microsoft::ReactNative::Composition::Experimental::UriBrushFactory>
-  GetSourceAsync(
+  winrt::Windows::Foundation::IAsyncOperation<winrt::Microsoft::ReactNative::Composition::ImageResponse>
+  GetImageResponseAsync(
       const winrt::Microsoft::ReactNative::IReactContext &context,
       const winrt::Microsoft::ReactNative::Composition::ImageSource &imageSource) {
-    co_return [uri = imageSource.Uri(), size = imageSource.Size(), scale = imageSource.Scale(), context](
-                  const winrt::Microsoft::ReactNative::IReactContext &reactContext,
-                  const winrt::Microsoft::ReactNative::Composition::Experimental::ICompositionContext
-                      &compositionContext) -> winrt::Microsoft::ReactNative::Composition::Experimental::IBrush {
-      auto compositor =
-          winrt::Microsoft::ReactNative::Composition::Experimental::MicrosoftCompositionContextHelper::InnerCompositor(
-              compositionContext);
-      auto drawingBrush = compositionContext.CreateDrawingSurfaceBrush(
-          size,
-          winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
-          winrt::Windows::Graphics::DirectX::DirectXAlphaMode::Premultiplied);
-      POINT pt;
-      Microsoft::ReactNative::Composition::AutoDrawDrawingSurface autoDraw(drawingBrush, scale, &pt);
-      auto renderTarget = autoDraw.GetRenderTarget();
-
-      winrt::com_ptr<ID2D1SolidColorBrush> brush;
-      renderTarget->CreateSolidColorBrush({1.0f, 0.0f, 0.0f, 1.0f}, brush.put());
-      renderTarget->DrawEllipse(
-          {{(pt.x + size.Width / 2) / scale, (pt.y + size.Height / 2) / scale},
-           (size.Width / 2) / scale,
-           (size.Height / 2) / scale},
-          brush.get());
-
-      return drawingBrush;
-    };
+    co_return winrt::Microsoft::ReactNative::Composition::Experimental::UriBrushFactoryImageResponse(
+        [uri = imageSource.Uri(), size = imageSource.Size(), scale = imageSource.Scale(), context](
+            const winrt::Microsoft::ReactNative::IReactContext &reactContext,
+            const winrt::Microsoft::ReactNative::Composition::Experimental::ICompositionContext &compositionContext)
+            -> winrt::Microsoft::ReactNative::Composition::Experimental::IBrush {
+          auto compositor = winrt::Microsoft::ReactNative::Composition::Experimental::
+              MicrosoftCompositionContextHelper::InnerCompositor(compositionContext);
+          auto drawingBrush = compositionContext.CreateDrawingSurfaceBrush(
+              size,
+              winrt::Windows::Graphics::DirectX::DirectXPixelFormat::B8G8R8A8UIntNormalized,
+              winrt::Windows::Graphics::DirectX::DirectXAlphaMode::Premultiplied);
+          POINT pt;
+          Microsoft::ReactNative::Composition::AutoDrawDrawingSurface autoDraw(drawingBrush, scale, &pt);
+          auto renderTarget = autoDraw.GetRenderTarget();
+
+          winrt::com_ptr<ID2D1SolidColorBrush> brush;
+          renderTarget->CreateSolidColorBrush({1.0f, 0.0f, 0.0f, 1.0f}, brush.put());
+          renderTarget->DrawEllipse(
+              {{(pt.x + size.Width / 2) / scale, (pt.y + size.Height / 2) / scale},
+               (size.Width / 2) / scale,
+               (size.Height / 2) / scale},
+              brush.get());
+
+          return drawingBrush;
+        });
   }
 };
 

vnext/Desktop.IntegrationTests/HttpResourceIntegrationTests.cpp

@@ -212,6 +212,8 @@ TEST_CLASS (HttpResourceIntegrationTest) {
     string error;
     IHttpResource::Response response;
 
+    MicrosoftReactSetRuntimeOptionString("Http.UserAgent", "React Native Windows");
+
     auto resource = IHttpResource::Make();
     resource->SetOnResponse([&rcPromise, &response](int64_t, IHttpResource::Response callbackResponse) {
       response = callbackResponse;
@@ -222,8 +224,6 @@ TEST_CLASS (HttpResourceIntegrationTest) {
       rcPromise.set_value();
     });
 
-    MicrosoftReactSetRuntimeOptionString("Http.UserAgent", "React Native Windows");
-
     //clang-format off
     resource->SendRequest(
         "GET",

vnext/Desktop.UnitTests/RedirectHttpFilterUnitTest.cpp

@@ -39,7 +39,7 @@ TEST_CLASS (RedirectHttpFilterUnitTest) {
   }
 
   TEST_METHOD(QueryInterfacesSucceeds) {
-    auto filter = winrt::make<RedirectHttpFilter>();
+    auto filter = winrt::make<RedirectHttpFilter>(winrt::hstring{});
 
     auto iFilter = filter.try_as<IHttpFilter>();
     Assert::IsFalse(iFilter == nullptr);
@@ -82,7 +82,7 @@ TEST_CLASS (RedirectHttpFilterUnitTest) {
       co_return response;
     };
 
-    auto filter = winrt::make<RedirectHttpFilter>(std::move(mockFilter1), std::move(mockFilter2));
+    auto filter = winrt::make<RedirectHttpFilter>(std::move(mockFilter1), std::move(mockFilter2), winrt::hstring{});
     auto client = HttpClient{filter};
     auto request = HttpRequestMessage{HttpMethod::Get(), Uri{url1}};
     auto sendOp = client.SendRequestAsync(request);
@@ -118,7 +118,7 @@ TEST_CLASS (RedirectHttpFilterUnitTest) {
       co_return response;
     };
 
-    auto filter = winrt::make<RedirectHttpFilter>(std::move(mockFilter1), std::move(mockFilter2));
+    auto filter = winrt::make<RedirectHttpFilter>(std::move(mockFilter1), std::move(mockFilter2), winrt::hstring{});
     // Disable automatic redirect
     filter.try_as<IHttpBaseProtocolFilter>().AllowAutoRedirect(false);
 
@@ -176,7 +176,8 @@ TEST_CLASS (RedirectHttpFilterUnitTest) {
       co_return response;
     };
 
-    auto filter = winrt::make<RedirectHttpFilter>(maxRedirects, std::move(mockFilter1), std::move(mockFilter2));
+    auto filter =
+        winrt::make<RedirectHttpFilter>(maxRedirects, std::move(mockFilter1), std::move(mockFilter2), winrt::hstring{});
     auto client = HttpClient{filter};
     auto request = HttpRequestMessage{HttpMethod::Get(), Uri{url1}};
     ResponseOperation sendOp = nullptr;

vnext/Microsoft.ReactNative/Fabric/Composition/CompositionRootView.cpp

@@ -685,13 +685,21 @@ void CompositionRootView::Arrange(
 }
 
 #ifdef USE_WINUI3
-winrt::Microsoft::UI::Content::ContentIsland CompositionRootView::Island() noexcept {
+winrt::Microsoft::UI::Content::ContentIsland CompositionRootView::Island() {
   if (!m_compositor) {
     return nullptr;
   }
 
   if (!m_island) {
-    auto rootVisual = m_compositor.CreateSpriteVisual();
+    winrt::Microsoft::UI::Composition::SpriteVisual rootVisual{nullptr};
+    try {
+      rootVisual = m_compositor.CreateSpriteVisual();
+    } catch (const winrt::hresult_error &e) {
+      // If the compositor has been shutdown, then we shouldn't attempt to initialize the island
+      if (e.code() == RO_E_CLOSED)
+        return nullptr;
+      throw e;
+    }
 
     InternalRootVisual(
         winrt::Microsoft::ReactNative::Composition::Experimental::MicrosoftCompositionContextHelper::CreateVisual(

vnext/Microsoft.ReactNative/Fabric/Composition/CompositionRootView.h

@@ -48,7 +48,7 @@ struct CompositionRootView
 
 #ifdef USE_WINUI3
   CompositionRootView(const winrt::Microsoft::UI::Composition::Compositor &compositor) noexcept;
-  winrt::Microsoft::UI::Content::ContentIsland Island() noexcept;
+  winrt::Microsoft::UI::Content::ContentIsland Island();
 #endif
 
   // property ReactViewHost

vnext/Microsoft.ReactNative/Fabric/Composition/TextInput/WindowsTextInputComponentView.cpp

@@ -1411,7 +1411,7 @@ winrt::com_ptr<::IDWriteTextLayout> WindowsTextInputComponentView::CreatePlaceho
 
 void WindowsTextInputComponentView::DrawText() noexcept {
   m_needsRedraw = true;
-  if (m_cDrawBlock || theme()->IsEmpty()) {
+  if (m_cDrawBlock || theme()->IsEmpty() || !m_textServices) {
     return;
   }
 

vnext/Microsoft.ReactNative/Fabric/Composition/UriImageManager.cpp

@@ -5,11 +5,19 @@
 #include "UriImageManager.h"
 
 #include "Composition.ImageSource.g.h"
+#include <Composition.Experimental.UriBrushFactoryImageResponse.g.cpp>
+#include <Composition.ImageFailedResponse.g.cpp>
+#include <Composition.ImageResponse.g.cpp>
+#include <Composition.StreamImageResponse.g.cpp>
+#include <Composition.UriBrushFactoryImageResponse.g.cpp>
 #include <AutoDraw.h>
+#include <IBlobPersistor.h>
+#include <Networking/NetworkPropertyIds.h>
 #include <ReactPropertyBag.h>
 #include <d2d1_3.h>
 #include <shcore.h>
 #include <winrt/Microsoft.ReactNative.Composition.Experimental.h>
+#include <winrt/Microsoft.ReactNative.Composition.h>
 #include <winrt/Windows.Security.Cryptography.h>
 #include <winrt/Windows.Storage.Streams.h>
 
@@ -54,16 +62,14 @@ winrt::Microsoft::ReactNative::Composition::ImageSource MakeImageSource(
  *   />
  *
  */
-struct SvgDataImageHandler : winrt::implements<
-                                 SvgDataImageHandler,
-                                 winrt::Microsoft::ReactNative::Composition::Experimental::IUriBrushProvider,
-                                 winrt::Microsoft::ReactNative::Composition::IUriImageProvider> {
+struct SvgDataImageHandler
+    : winrt::implements<SvgDataImageHandler, winrt::Microsoft::ReactNative::Composition::IUriImageProvider> {
   bool CanLoadImageUri(winrt::Microsoft::ReactNative::IReactContext context, winrt::Windows::Foundation::Uri uri) {
     return uri.SchemeName() == L"data" && std::wstring_view(uri.Path()).starts_with(L"image/svg+xml;base64,");
   }
 
-  winrt::Windows::Foundation::IAsyncOperation<winrt::Microsoft::ReactNative::Composition::Experimental::UriBrushFactory>
-  GetSourceAsync(
+  winrt::Windows::Foundation::IAsyncOperation<winrt::Microsoft::ReactNative::Composition::ImageResponse>
+  GetImageResponseAsync(
       const winrt::Microsoft::ReactNative::IReactContext &context,
       const winrt::Microsoft::ReactNative::Composition::ImageSource &imageSource) {
     auto path = winrt::to_string(imageSource.Uri().Path());
@@ -85,7 +91,7 @@ struct SvgDataImageHandler : winrt::implements<
       co_await memoryStream.WriteAsync(buffer);
       memoryStream.Seek(0);
 
-      co_return
+      co_return winrt::Microsoft::ReactNative::Composition::Experimental::UriBrushFactoryImageResponse(
           [memoryStream, size, scale](
               const winrt::Microsoft::ReactNative::IReactContext &reactContext,
               const winrt::Microsoft::ReactNative::Composition::Experimental::ICompositionContext &compositionContext)
@@ -126,12 +132,13 @@ struct SvgDataImageHandler : winrt::implements<
             renderTarget->SetTransform(originalTransform);
 
             return drawingBrush;
-          };
+          });
 
-    } catch (winrt::hresult_error const &) {
+    } catch (winrt::hresult_error const &ex) {
+      co_return winrt::Microsoft::ReactNative::Composition::ImageFailedResponse(ex.message());
     }
 
-    co_return nullptr;
+    winrt::throw_hresult(E_UNEXPECTED);
   }
 };
 
@@ -145,15 +152,14 @@ struct SvgDataImageHandler : winrt::implements<
  *   />
  *
  */
-struct DataImageHandler : winrt::implements<
-                              DataImageHandler,
-                              winrt::Microsoft::ReactNative::Composition::IUriImageStreamProvider,
-                              winrt::Microsoft::ReactNative::Composition::IUriImageProvider> {
+struct DataImageHandler
+    : winrt::implements<DataImageHandler, winrt::Microsoft::ReactNative::Composition::IUriImageProvider> {
   bool CanLoadImageUri(winrt::Microsoft::ReactNative::IReactContext context, winrt::Windows::Foundation::Uri uri) {
     return uri.SchemeName() == L"data";
   }
 
-  winrt::Windows::Foundation::IAsyncOperation<winrt::Windows::Storage::Streams::IRandomAccessStream> GetSourceAsync(
+  winrt::Windows::Foundation::IAsyncOperation<winrt::Microsoft::ReactNative::Composition::ImageResponse>
+  GetImageResponseAsync(
       const winrt::Microsoft::ReactNative::IReactContext &context,
       const winrt::Microsoft::ReactNative::Composition::ImageSource &imageSource) {
     auto path = winrt::to_string(imageSource.Uri().Path());
@@ -173,12 +179,57 @@ struct DataImageHandler : winrt::implements<
       co_await memoryStream.WriteAsync(buffer);
       memoryStream.Seek(0);
 
-      co_return memoryStream;
+      co_return winrt::Microsoft::ReactNative::Composition::StreamImageResponse(memoryStream);
     } catch (winrt::hresult_error const &) {
       // Base64 decode failed
+      co_return winrt::Microsoft::ReactNative::Composition::ImageFailedResponse(
+          L"Invalid base64 encoding in inline image data");
+    }
+
+    winrt::throw_hresult(E_UNEXPECTED);
+  }
+};
+
+/**
+ * This ImageHandler will handle uri loading data from blobs
+ *
+ *   <Image
+ *      style={{width: 400, height: 200}}
+ *      source={{uri:'blob:<guid>?offset=<offset>&size=<size>'}}
+ *   />
+ *
+ */
+struct BlobImageHandler
+    : winrt::implements<BlobImageHandler, winrt::Microsoft::ReactNative::Composition::IUriImageProvider> {
+  bool CanLoadImageUri(winrt::Microsoft::ReactNative::IReactContext context, winrt::Windows::Foundation::Uri uri) {
+    return uri.SchemeName() == L"blob";
+  }
+
+  winrt::Windows::Foundation::IAsyncOperation<winrt::Microsoft::ReactNative::Composition::ImageResponse>
+  GetImageResponseAsync(
+      const winrt::Microsoft::ReactNative::IReactContext &context,
+      const winrt::Microsoft::ReactNative::Composition::ImageSource &imageSource) {
+    if (auto prop = winrt::Microsoft::ReactNative::ReactPropertyBag(context.Properties())
+                        .Get(::Microsoft::React::BlobModulePersistorPropertyId())) {
+      auto weakBlobPersistor = prop.Value();
+      if (auto persistor = weakBlobPersistor.lock()) {
+        auto queryParsed = imageSource.Uri().QueryParsed();
+        auto guid = winrt::to_string(imageSource.Uri().Path());
+        int64_t offset = _atoi64(winrt::to_string(queryParsed.GetFirstValueByName(L"offset")).c_str());
+        int64_t size = _atoi64(winrt::to_string(queryParsed.GetFirstValueByName(L"size")).c_str());
+
+        auto arr = persistor->ResolveMessage(std::move(guid), offset, size);
+        winrt::Windows::Storage::Streams::InMemoryRandomAccessStream memoryStream;
+        winrt::Windows::Storage::Streams::DataWriter dataWriter{memoryStream};
+        dataWriter.WriteBytes(arr);
+        co_await dataWriter.StoreAsync();
+        memoryStream.Seek(0);
+
+        co_return winrt::Microsoft::ReactNative::Composition::StreamImageResponse(memoryStream.CloneStream());
+      }
     }
 
-    co_return nullptr;
+    co_return winrt::Microsoft::ReactNative::Composition::ImageFailedResponse(L"Failed to load image from blob");
   }
 };
 
@@ -191,6 +242,7 @@ static const ReactPropertyId<ReactNonAbiValue<std::shared_ptr<UriImageManager>>>
 UriImageManager::UriImageManager() {
   m_providers.push_back(winrt::make<SvgDataImageHandler>());
   m_providers.push_back(winrt::make<DataImageHandler>());
+  m_providers.push_back(winrt::make<BlobImageHandler>());
 }
 
 void UriImageManager::Install(
@@ -227,4 +279,23 @@ IUriImageProvider UriImageManager::TryGetUriImageProvider(
   return nullptr;
 }
 
+ImageResponseOrImageErrorInfo ImageResponse::ResolveImage() {
+  winrt::throw_hresult(E_NOTIMPL);
+}
+
+ImageResponseOrImageErrorInfo ImageFailedResponse::ResolveImage() {
+  ImageResponseOrImageErrorInfo imageOrError;
+  imageOrError.errorInfo = std::make_shared<facebook::react::ImageErrorInfo>();
+  imageOrError.errorInfo->responseCode = static_cast<int>(m_statusCode);
+  imageOrError.errorInfo->error = winrt::to_string(m_errorMessage);
+  if (imageOrError.errorInfo->error.empty()) {
+    imageOrError.errorInfo->error = "Failed to load image.";
+  }
+  for (auto &&[header, value] : m_responseHeaders) {
+    imageOrError.errorInfo->httpResponseHeaders.push_back(
+        std::make_pair<std::string, std::string>(winrt::to_string(header), winrt::to_string(value)));
+  }
+  return imageOrError;
+}
+
 } // namespace winrt::Microsoft::ReactNative::Composition::implementation