Enable ViamImage image provider

example/viam_robot_example_app/lib/main.dart

@@ -55,12 +55,14 @@ class _MyHomePageState extends State<MyHomePage> {
     });
 
     // Be sure to create a .env file with these fields
+    final opts = RobotClientOptions.withApiKey(
+      dotenv.env['API_KEY_ID']!,
+      dotenv.env['API_KEY']!,
+    );
+    opts.dialOptions.attemptMdns = false;
     _robot = await RobotClient.atAddress(
       dotenv.env['ROBOT_LOCATION']!,
-      RobotClientOptions.withApiKey(
-        dotenv.env['API_KEY_ID']!,
-        dotenv.env['API_KEY']!,
-      ),
+      opts,
     );
 
     final services = _robot.resourceNames.where((element) => element.type == resourceTypeService);

example/viam_robot_example_app/lib/screens/stream.dart

@@ -1,8 +1,4 @@
-import 'dart:typed_data';
-import 'dart:ui' as ui;
-
 import 'package:flutter/material.dart';
-import 'package:image/image.dart' as img;
 import 'package:viam_sdk/viam_sdk.dart';
 import 'package:viam_sdk/widgets.dart';
 
@@ -21,57 +17,27 @@ class StreamScreen extends StatefulWidget {
 
 class _StreamScreenState extends State<StreamScreen> {
   // Single frame
-  ByteData? imageBytes;
-  bool _imgLoaded = false;
+  late Future<ViamImage> _viamImage;
+
+  @override
+  void initState() {
+    super.initState();
+    _getImage();
+  }
 
   void _getImage() {
     setState(() {
-      _imgLoaded = false;
-    });
-    final imageFut = widget.camera.image();
-    imageFut.then((value) {
-      final convertFut = convertImageToFlutterUi(value.image ?? img.Image.empty());
-      convertFut.then((value) {
-        final pngFut = value.toByteData(format: ui.ImageByteFormat.png);
-        pngFut.then((value) => setState(() {
-              imageBytes = value;
-              _imgLoaded = true;
-            }));
-      });
+      _viamImage = widget.camera.image();
     });
   }
 
-  Future<ui.Image> convertImageToFlutterUi(img.Image image) async {
-    if (image.format != img.Format.uint8 || image.numChannels != 4) {
-      final cmd = img.Command()
-        ..image(image)
-        ..convert(format: img.Format.uint8, numChannels: 4);
-      final rgba8 = await cmd.getImageThread();
-      if (rgba8 != null) {
-        image = rgba8;
-      }
-    }
-
-    final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
-
-    final ui.ImageDescriptor id =
-        ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888);
-
-    final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width);
-
-    final ui.FrameInfo fi = await codec.getNextFrame();
-    final ui.Image uiImage = fi.image;
-
-    return uiImage;
-  }
-
   @override
   Widget build(BuildContext context) {
     return Scaffold(
       appBar: AppBar(
         title: Text(widget.resourceName.name.toUpperCase()),
       ),
-      body: Center(
+      body: SingleChildScrollView(
         child: Column(
           children: [
             const SizedBox(height: 16),
@@ -82,7 +48,10 @@ class _StreamScreenState extends State<StreamScreen> {
             const SizedBox(height: 16),
             ViamCameraStreamView(camera: widget.camera, streamClient: widget.client),
             const SizedBox(height: 16),
-            if (_imgLoaded) Image.memory(Uint8List.view(imageBytes!.buffer), scale: 3),
+            FutureBuilder(
+                future: _viamImage,
+                builder: ((context, snapshot) =>
+                    snapshot.hasData ? Image(image: snapshot.data!.imageProvider) : const CircularProgressIndicator.adaptive())),
             const SizedBox(height: 16),
             ElevatedButton(
               child: const Text('Get image'),

example/viam_robot_example_app/macos/Podfile

@@ -1,4 +1,4 @@
-platform :osx, '10.14'
+platform :osx, '11'
 
 # CocoaPods analytics sends network stats synchronously affecting flutter build latency.
 ENV['COCOAPODS_DISABLE_STATS'] = 'true'

example/viam_robot_example_app/macos/Runner.xcodeproj/project.pbxproj

@@ -56,7 +56,7 @@
 		15242523DB1FCDEE32F203C2 /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; };
 		333000ED22D3DE5D00554162 /* Warnings.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Warnings.xcconfig; sourceTree = "<group>"; };
 		335BBD1A22A9A15E00E9071D /* GeneratedPluginRegistrant.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = GeneratedPluginRegistrant.swift; sourceTree = "<group>"; };
-		33CC10ED2044A3C60003C045 /* viam_example_app.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = viam_example_app.app; sourceTree = BUILT_PRODUCTS_DIR; };
+		33CC10ED2044A3C60003C045 /* Viam Example App.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Viam Example App.app"; sourceTree = BUILT_PRODUCTS_DIR; };
 		33CC10F02044A3C60003C045 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = "<group>"; };
 		33CC10F22044A3C60003C045 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; name = Assets.xcassets; path = Runner/Assets.xcassets; sourceTree = "<group>"; };
 		33CC10F52044A3C60003C045 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = Base; path = Base.lproj/MainMenu.xib; sourceTree = "<group>"; };
@@ -112,7 +112,7 @@
 		33CC10EE2044A3C60003C045 /* Products */ = {
 			isa = PBXGroup;
 			children = (
-				33CC10ED2044A3C60003C045 /* viam_example_app.app */,
+				33CC10ED2044A3C60003C045 /* Viam Example App.app */,
 			);
 			name = Products;
 			sourceTree = "<group>";
@@ -159,7 +159,6 @@
 				670B361EC221A983BEC9B00C /* Pods-Runner.release.xcconfig */,
 				CD7EFC0D75004BBCC6F28023 /* Pods-Runner.profile.xcconfig */,
 			);
-			name = Pods;
 			path = Pods;
 			sourceTree = "<group>";
 		};
@@ -193,7 +192,7 @@
 			);
 			name = Runner;
 			productName = Runner;
-			productReference = 33CC10ED2044A3C60003C045 /* viam_example_app.app */;
+			productReference = 33CC10ED2044A3C60003C045 /* Viam Example App.app */;
 			productType = "com.apple.product-type.application";
 		};
 /* End PBXNativeTarget section */
@@ -427,6 +426,7 @@
 					"$(inherited)",
 					"@executable_path/../Frameworks",
 				);
+				MACOSX_DEPLOYMENT_TARGET = 11.0;
 				PROVISIONING_PROFILE_SPECIFIER = "";
 				SWIFT_VERSION = 5.0;
 			};
@@ -553,6 +553,7 @@
 					"$(inherited)",
 					"@executable_path/../Frameworks",
 				);
+				MACOSX_DEPLOYMENT_TARGET = 11.0;
 				PROVISIONING_PROFILE_SPECIFIER = "";
 				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
 				SWIFT_VERSION = 5.0;
@@ -573,6 +574,7 @@
 					"$(inherited)",
 					"@executable_path/../Frameworks",
 				);
+				MACOSX_DEPLOYMENT_TARGET = 11.0;
 				PROVISIONING_PROFILE_SPECIFIER = "";
 				SWIFT_VERSION = 5.0;
 			};

example/viam_robot_example_app/macos/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme

@@ -15,7 +15,7 @@
             <BuildableReference
                BuildableIdentifier = "primary"
                BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-               BuildableName = "viam_example_app.app"
+               BuildableName = "Viam Example App.app"
                BlueprintName = "Runner"
                ReferencedContainer = "container:Runner.xcodeproj">
             </BuildableReference>
@@ -31,7 +31,7 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-            BuildableName = "viam_example_app.app"
+            BuildableName = "Viam Example App.app"
             BlueprintName = "Runner"
             ReferencedContainer = "container:Runner.xcodeproj">
          </BuildableReference>
@@ -54,7 +54,7 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-            BuildableName = "viam_example_app.app"
+            BuildableName = "Viam Example App.app"
             BlueprintName = "Runner"
             ReferencedContainer = "container:Runner.xcodeproj">
          </BuildableReference>
@@ -71,7 +71,7 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "33CC10EC2044A3C60003C045"
-            BuildableName = "viam_example_app.app"
+            BuildableName = "Viam Example App.app"
             BlueprintName = "Runner"
             ReferencedContainer = "container:Runner.xcodeproj">
          </BuildableReference>

lib/src/media/image.dart

@@ -89,7 +89,7 @@ class MimeType {
   }
 }
 
-/// A custom image type that contains the [MimeTYpe], raw image data, and lazily loads and caches an [img.Image].
+/// A custom image type that contains the [MimeType], raw image data, and lazily loads and caches an [img.Image].
 class ViamImage {
   /// The mimetype of the image
   final MimeType mimeType;

lib/widgets.dart

@@ -1,5 +1,6 @@
 export 'widgets/button.dart';
 export 'widgets/camera_stream.dart';
+export 'widgets/image_view.dart';
 export 'widgets/joystick.dart';
 export 'widgets/multi_camera_stream.dart';
 export 'widgets/refreshable_data_table.dart';

lib/widgets/image_view.dart

@@ -0,0 +1,103 @@
+import 'dart:async';
+import 'dart:ui' as ui;
+
+import 'package:flutter/foundation.dart';
+import 'package:flutter/material.dart';
+import 'package:image/image.dart' as img;
+import 'package:viam_sdk/viam_sdk.dart';
+
+/// Display a ViamImage inside Flutter's [Image] widget using this provider
+/// as the image value.
+///
+/// ```dart
+/// Image(image: viamImage.imageProvider);
+/// ```
+class ViamImageProvider extends ImageProvider<ViamImage> {
+  const ViamImageProvider(this.image);
+
+  final ViamImage image;
+
+  @override
+  Future<ViamImage> obtainKey(ImageConfiguration configuration) {
+    return SynchronousFuture(image);
+  }
+
+  @override
+  ImageStreamCompleter loadImage(ViamImage key, ImageDecoderCallback decode) {
+    return MultiFrameImageStreamCompleter(
+        codec: key.uiData
+            .then<Uint8List>((value) {
+              if (value == null) {
+                throw Exception('Unable to convert image to displayable format');
+              }
+              return value;
+            })
+            .catchError((Object e, StackTrace stack) {
+              scheduleMicrotask(() {
+                PaintingBinding.instance.imageCache.evict(key);
+              });
+              return Future<Uint8List>.error(e, stack);
+            })
+            .then<ui.ImmutableBuffer>(ui.ImmutableBuffer.fromUint8List)
+            .then<ui.Codec>(decode),
+        scale: 1.0);
+  }
+}
+
+extension UI on img.Image {
+  /// Data that allows [img.Image] to be compatible with dart:ui or Flutter.
+  ///
+  /// ```dart
+  /// img.Image image = ...;
+  /// final imgData = await image.uiData;
+  /// Image.memory(imgData);
+  /// ```
+  Future<Uint8List> get uiData async {
+    img.Image image = this;
+    if (image.format != img.Format.uint8 || image.numChannels != 4) {
+      final cmd = img.Command()
+        ..image(this)
+        ..convert(format: img.Format.uint8, numChannels: 4);
+      final rgba8 = await cmd.getImageThread();
+      if (rgba8 != null) {
+        image = rgba8;
+      }
+    }
+
+    final ui.ImmutableBuffer buffer = await ui.ImmutableBuffer.fromUint8List(image.toUint8List());
+
+    final ui.ImageDescriptor id =
+        ui.ImageDescriptor.raw(buffer, height: image.height, width: image.width, pixelFormat: ui.PixelFormat.rgba8888);
+
+    final ui.Codec codec = await id.instantiateCodec(targetHeight: image.height, targetWidth: image.width);
+
+    final ui.FrameInfo fi = await codec.getNextFrame();
+    final ui.Image uiImage = fi.image;
+
+    final png = await uiImage.toByteData(format: ui.ImageByteFormat.png);
+    return Uint8List.view(png!.buffer);
+  }
+}
+
+extension Flutter on ViamImage {
+  /// Data that allows [ViamImage] to be compatible with dart:ui or Flutter.
+  ///
+  /// ```dart
+  /// ViamImage image = ...;
+  /// final imgData = await image.uiData;
+  /// Image.memory(imgData);
+  /// ```
+  Future<Uint8List?> get uiData async {
+    return await image?.uiData;
+  }
+
+  /// Display a ViamImage inside Flutter's [Image] widget using this provider
+  /// as the image value.
+  ///
+  /// ```dart
+  /// Image(image: viamImage.imageProvider);
+  /// ```
+  ImageProvider get imageProvider {
+    return ViamImageProvider(this);
+  }
+}