diff --git a/sksurgeryvtk/models/voxelise.py b/sksurgeryvtk/models/voxelise.py index 96d64129..89a3e4d8 100644 --- a/sksurgeryvtk/models/voxelise.py +++ b/sksurgeryvtk/models/voxelise.py @@ -606,7 +606,8 @@ def apply_displacement_to_mesh(mesh: Union[vtk.vtkDataObject, str], 0, vtk.vtkDataObject.FIELD_ASSOCIATION_POINTS, "preoperativeSurface") - threshold.ThresholdByLower(0) + threshold.SetLowerThreshold(0) + threshold.SetThresholdFunction(threshold.THRESHOLD_LOWER) threshold.SetInputData(field) threshold.Update() fieldInternal = threshold.GetOutput() diff --git a/sksurgeryvtk/models/vtk_grid_model.py b/sksurgeryvtk/models/vtk_grid_model.py index 9927dc3f..9ea24870 100644 --- a/sksurgeryvtk/models/vtk_grid_model.py +++ b/sksurgeryvtk/models/vtk_grid_model.py @@ -125,5 +125,7 @@ def threshold_between(self, lower: float, upper: float): :param upper: Upper limit :type upper: float """ - self.threshold.ThresholdBetween(lower, upper) + self.threshold.SetLowerThreshold(lower) + self.threshold.SetUpperThreshold(upper) + self.threshold.SetThresholdFunction(self.threshold.THRESHOLD_BETWEEN) self.threshold.Update() diff --git a/sksurgeryvtk/widgets/vtk_overlay_window.py b/sksurgeryvtk/widgets/vtk_overlay_window.py index 3d50036b..73ec0cbb 100644 --- a/sksurgeryvtk/widgets/vtk_overlay_window.py +++ b/sksurgeryvtk/widgets/vtk_overlay_window.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- +# pylint: disable=too-many-instance-attributes, no-name-in-module +# pylint:disable=super-with-arguments, too-many-arguments, line-too-long + """ -Module to provide a VTK scene on top of a video stream, -thereby enabling a basic augmented reality viewer. +Module to provide a set of VTK renderers that can be used to create an Augmented Reality viewer. Expected usage: @@ -11,19 +13,17 @@ window = VTKOverlayWindow() window.add_vtk_models(list) # list of VTK models window.add_vtk_actor(actor) # or individual actor - window.set_camera_matrix(ndarray) # Set 3x3 ndarray of camera matrix + window.set_camera_matrix(ndarray) # Set 3x3 ndarray of OpenCV camera intrinsic matrix. while True: - image = # acquire np.ndarray image some how - window.set_video_image(image) + image = # acquire np.ndarray image some how, e.g. from webcam or USB source. + window.set_video_image(image) window.set_camera_pose(camera_to_world) # set 4x4 ndarray """ -# pylint: disable=too-many-instance-attributes, no-name-in-module -# pylint:disable=super-with-arguments import logging import cv2 @@ -44,113 +44,177 @@ class VTKOverlayWindow(QVTKRenderWindowInteractor): """ Sets up a VTK Overlay Window that can be used to overlay multiple VTK models on a video stream. Internally, the Window - has 3 renderers. The background renderer displays - the video image in the background. The foreground renderer - displays a VTK scene overlaid on the background. If you make your - VTK models semi-transparent you get a merging effect. - An additional rendering layer is just for overlays - like picture-in-picture ultrasound. + has 5 renderers, 0=backmost, 5=frontmost. + + # Layer 0: Video + # Layer 1: VTK rendered models - e.g. internal anatomy + # Layer 2: Video + # Layer 3: VTK rendered models - e.g. external anatomy + # Layer 4: VTK rendered text annotations. + + The video channels should be RGBA. You can choose in the constructor + whether the video should go to Layer 0 or Layer 2. + + If you put video in the Layer 0, and overlay models in Layer 1, you get a simple overlay + which may be suitable for things like overlaying calibration points on chessboards, + but you will get poor visual coherence for medical AR, as the bright colours of a synthetic overlay + will always make the model appear in front of the video, even when you dial back the opacity of the model. + + If you put the video in Layer 2, it would obscure the rendered models in Layer 1. + But you can apply a mask to the alpha channel setting the alpha to either 0 or 255. + If the mask contains say a circle, it will have the effect of showing the video when the + alpha channel is 255 and the rendering behind when the alpha channel is 0. So, you can + use this to peek inside an organ by rendering the video in Layer 2 with a mask creating a hole, + and the internal anatomy in Layer 1. Then you put the external anatomy, e.g. liver surface in Layer 3. :param offscreen: Enable/Disable offscreen rendering. :param camera_matrix: Camera extrinsics matrix. :param clipping_range: Near/Far clipping range. - :param zbuffer: if True, will only render zbuffer of main renderer. - :param opencv_style: If True, adopts OpenCV convention, otherwise OpenGL. + :param zbuffer: If True, will only render zbuffer of main renderer. + :param opencv_style: If True, adopts OpenCV camera convention, otherwise OpenGL. :param init_pose: If True, will initialise the camera pose to identity. :param reset_camera: If True, resets camera when a new model is added. :param init_widget: If True we will call self.Initialize and self.Start as part of the init function. Set to false if you're on Linux. + :param video_in_layer_0: If true, will add video to Layer 0, fully opaque, no masking. + :param video_in_layer_2: If true, will add video to Layer 1. If layer_2_video_mask is present, will mask alpha channel. """ - def __init__(self, - offscreen=False, - camera_matrix=None, - clipping_range=(1, 1000), - zbuffer=False, - opencv_style=True, - init_pose=False, - reset_camera=True, - init_widget=True - ): + def __init__( + self, + offscreen=False, + camera_matrix=None, + clipping_range=(1, 1000), + zbuffer=False, + opencv_style=True, + init_pose=False, + reset_camera=True, + init_widget=True, + video_in_layer_0=True, # For backwards compatibility, prior to 3rd Feb 2024. + video_in_layer_2=False, # For backwards compatibility, prior to 3rd Feb 2024. + layer_2_video_mask=None, # For masking in Layer 3 + use_depth_peeling=True # Historically, has defaulted to true. + ): """ Constructs a new VTKOverlayWindow. """ super(VTKOverlayWindow, self).__init__() + # Take and cache/store constructor arguments. if offscreen: self.GetRenderWindow().SetOffScreenRendering(1) else: self.GetRenderWindow().SetOffScreenRendering(0) - self.camera_matrix = camera_matrix - self.camera_to_world = np.eye(4) self.clipping_range = clipping_range - self.aspect_ratio = 1 self.zbuffer = zbuffer - self.reset_camera = reset_camera self.opencv_style = opencv_style + self.reset_camera = reset_camera + self.video_in_layer_0 = video_in_layer_0 + self.video_in_layer_2 = video_in_layer_2 + self.layer_2_video_mask = layer_2_video_mask - self.input = np.ones((400, 400, 3), dtype=np.uint8) + # Some default reference data, or member variables. + self.aspect_ratio = 1 + self.camera_to_world = np.eye(4) + self.rgb_input = np.ones((400, 400, 3), dtype=np.uint8) self.rgb_frame = None + self.rgba_frame = None self.screen = None + self.mask_image = None # VTK objects initialised later - self.foreground_renderer = None - self.image_importer = None - self.background_shape = None - self.image_importer = None - self.image_extent = None - self.background_actor = None - self.background_renderer = None - self.background_camera = None self.output = None self.output_halved = None self.vtk_image = None self.vtk_array = None self.interactor = None - # Enable VTK Depth peeling settings for render window. - self.GetRenderWindow().AlphaBitPlanesOn() - self.GetRenderWindow().SetMultiSamples(0) - - # Three layers used, one for the background, one for VTK models, - # and one for other overlay (e.g. text) - self.GetRenderWindow().SetNumberOfLayers(3) - - # Use an image importer to import the video image. - self.background_shape = self.input.shape - self.image_extent = (0, self.background_shape[1] - 1, - 0, self.background_shape[0] - 1, 0, 0) - self.image_importer = vtk.vtkImageImport() - self.image_importer.SetDataScalarTypeToUnsignedChar() - self.image_importer.SetNumberOfScalarComponents(3) - self.image_importer.SetDataExtent(self.image_extent) - self.image_importer.SetWholeExtent(self.image_extent) - - self.set_video_image(self.input) - - # Create and setup background (video) renderer. - self.background_actor = vtk.vtkImageActor() - self.background_actor.SetInputData(self.image_importer.GetOutput()) - self.background_actor.VisibilityOff() - self.background_renderer = vtk.vtkRenderer() - self.background_renderer.SetLayer(0) - self.background_renderer.InteractiveOff() - self.background_renderer.AddActor(self.background_actor) - self.background_camera = self.background_renderer.GetActiveCamera() - self.background_camera.ParallelProjectionOn() - - # Create and setup foreground (VTK scene) renderer. - self.foreground_renderer = vtk.vtkRenderer() - self.foreground_renderer.SetLayer(1) - self.foreground_renderer.UseDepthPeelingOn() - self.foreground_renderer.SetMaximumNumberOfPeels(100) - self.foreground_renderer.SetOcclusionRatio(0.1) - self.foreground_renderer.LightFollowCameraOn() - - # Crate and setup generic overlay renderer. - self.generic_overlay_renderer = vtk.vtkRenderer() - self.generic_overlay_renderer.SetLayer(2) + # Setup an image importer to import the RGB video image. + # Until the image is set, we use the default one created above. + self.rgb_image_extent = ( + 0, + self.rgb_input.shape[1] - 1, + 0, + self.rgb_input.shape[0] - 1, + 0, + self.rgb_input.shape[2] - 1, + ) + self.rgb_image_importer = vtk.vtkImageImport() + self.rgb_image_importer.SetDataScalarTypeToUnsignedChar() + self.rgb_image_importer.SetNumberOfScalarComponents(3) + self.rgb_image_importer.SetDataExtent(self.rgb_image_extent) + self.rgb_image_importer.SetWholeExtent(self.rgb_image_extent) + + # Setup an image importer to import the RGBA video image. + # Until the image is set, we use the default one created above. + self.rgba_image_extent = ( + 0, + self.rgb_input.shape[1] - 1, + 0, + self.rgb_input.shape[0] - 1, + 0, + self.rgb_input.shape[2], + ) + self.rgba_image_importer = vtk.vtkImageImport() + self.rgba_image_importer.SetDataScalarTypeToUnsignedChar() + self.rgba_image_importer.SetNumberOfScalarComponents(4) + self.rgba_image_importer.SetDataExtent(self.rgba_image_extent) + self.rgba_image_importer.SetWholeExtent(self.rgba_image_extent) + + # Five layers used, see class level docstring. + self.GetRenderWindow().SetNumberOfLayers(5) + + # Create and setup layer 0 (video) renderer. + self.layer_0_image_actor = vtk.vtkImageActor() + self.layer_0_image_actor.SetInputData(self.rgb_image_importer.GetOutput()) + self.layer_0_image_actor.VisibilityOff() + self.layer_0_renderer = vtk.vtkRenderer() + self.layer_0_renderer.SetLayer(0) + self.layer_0_renderer.InteractiveOff() + self.layer_0_renderer.AddActor(self.layer_0_image_actor) + self.layer_0_camera = self.layer_0_renderer.GetActiveCamera() + self.layer_0_camera.ParallelProjectionOn() + + # Create and setup layer 1 (VTK scene) renderer. + self.layer_1_renderer = vtk.vtkRenderer() + self.layer_1_renderer.SetLayer(1) + self.layer_1_renderer.LightFollowCameraOn() + + # Create and setup layer 2 (masked video) renderer. + self.layer_2_image_actor = vtk.vtkImageActor() + self.layer_2_image_actor.SetInputData(self.rgba_image_importer.GetOutput()) + self.layer_2_image_actor.VisibilityOff() + self.layer_2_renderer = vtk.vtkRenderer() + self.layer_2_renderer.SetLayer(2) + self.layer_2_renderer.InteractiveOff() + self.layer_2_renderer.AddActor(self.layer_2_image_actor) + self.layer_2_camera = self.layer_2_renderer.GetActiveCamera() + self.layer_2_camera.ParallelProjectionOn() + + # Create and setup layer 3 (VTK scene) renderer. + self.layer_3_renderer = vtk.vtkRenderer() + self.layer_3_renderer.SetLayer(3) + self.layer_3_renderer.LightFollowCameraOn() + + # Create and setup layer 4 (Overlay's, like text annotations) renderer. + self.layer_4_renderer = vtk.vtkRenderer() + self.layer_4_renderer.SetLayer(4) + self.layer_4_renderer.LightFollowCameraOn() + + # Enable VTK Depth peeling settings for render window, and renderers. + if use_depth_peeling: + self.GetRenderWindow().AlphaBitPlanesOn() + self.GetRenderWindow().SetMultiSamples(0) + self.layer_1_renderer.UseDepthPeelingOn() + self.layer_1_renderer.SetMaximumNumberOfPeels(100) + self.layer_1_renderer.SetOcclusionRatio(0.1) + self.layer_3_renderer.UseDepthPeelingOn() + self.layer_3_renderer.SetMaximumNumberOfPeels(100) + self.layer_3_renderer.SetOcclusionRatio(0.1) + + # Use this to ensure the video is setup correctly at construction. + self.set_video_image(self.rgb_input) # Setup the general interactor style. See VTK docs for alternatives. self.interactor = vtk.vtkInteractorStyleTrackballCamera() @@ -162,15 +226,16 @@ def __init__(self, # foreground objects using RenderWindowInteractor, the foreground # should be added last. if not self.zbuffer: - self.GetRenderWindow().AddRenderer(self.background_renderer) - self.GetRenderWindow().AddRenderer(self.generic_overlay_renderer) - self.GetRenderWindow().AddRenderer(self.foreground_renderer) + self.GetRenderWindow().AddRenderer(self.layer_0_renderer) + self.GetRenderWindow().AddRenderer(self.layer_1_renderer) + self.GetRenderWindow().AddRenderer(self.layer_2_renderer) + self.GetRenderWindow().AddRenderer(self.layer_3_renderer) + self.GetRenderWindow().AddRenderer(self.layer_4_renderer) else: - self.GetRenderWindow().AddRenderer(self.foreground_renderer) + self.GetRenderWindow().AddRenderer(self.layer_1_renderer) # Set Qt Size Policy - self.size_policy = \ - QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) + self.size_policy = QSizePolicy(QSizePolicy.Expanding, QSizePolicy.Expanding) self.setSizePolicy(self.size_policy) # Set default position to origin. @@ -180,60 +245,128 @@ def __init__(self, # Startup the widget if init_widget: - self.Initialize() #Allows the interactor to initialize itself. - self.Start() #Start the event loop. + self.Initialize() # Allows the interactor to initialize itself. + self.Start() # Start the event loop. else: - print("\nYou've elected to initialize the VTKOverlayWindow(),", - "be sure to do it in your calling function.") + print( + "\nYou've elected to initialize the VTKOverlayWindow(),", + "be sure to do it in your calling function.", + ) def closeEvent(self, evt): super().closeEvent(evt) self.Finalize() + def set_video_mask(self, mask_image): + """ + Allows you to store a mask image, for alpha blending with any video. + Must be grey-scale, so 1 channel. + """ + if not isinstance(mask_image, np.ndarray): + raise TypeError("Input is not an np.ndarray") + if len(mask_image.shape) != 3: + raise ValueError( + "Input image should have X size, Y size and a single channel, e.g. grey scale." + ) + if mask_image.shape[2] != 1: + raise ValueError("Input image should be 1 channel, i.e. grey scale.") + self.mask_image = mask_image + def set_video_image(self, input_image): """ - Set the video image that is used for the background. + Sets the video image that is used for the background. + See also constructor args video_in_layer_0 and video_in_layer_2 which controls + in which layer(s) the video image ends up. """ if not isinstance(input_image, np.ndarray): - raise TypeError('Input is not an np.ndarray') - - if self.input.shape != input_image.shape: - self.background_actor.VisibilityOn() - self.background_shape = input_image.shape - self.image_extent = (0, self.background_shape[1] - 1, - 0, self.background_shape[0] - 1, 0, 0) - self.image_importer.SetDataExtent(self.image_extent) - self.image_importer.SetWholeExtent(self.image_extent) - self.__update_video_image_camera() - self.__update_projection_matrix() - - self.input = input_image - self.rgb_frame = np.copy(self.input[:, :, ::-1]) - self.image_importer.SetImportVoidPointer(self.rgb_frame.data) - self.image_importer.SetDataExtent(self.image_extent) - self.image_importer.SetWholeExtent(self.image_extent) - self.image_importer.Modified() - self.image_importer.Update() - - def __update_video_image_camera(self): - """ - Position the background renderer camera, so that the video image - is maximised and centralised in the screen. + raise TypeError("Input is not an np.ndarray") + if len(input_image.shape) != 3: + raise ValueError( + "Input image should have X size, Y size and a number of channels, e.g. RGB." + ) + if input_image.shape[2] != 3: + raise ValueError("Input image should be 3 channel, i.e. RGB.") + + # Note: We will assume that any video comming in is 3 channel, BGR. + # But layer 2 will use RGBA as we need the alpha channel. + + if ( + self.video_in_layer_0 and self.rgb_input.shape != input_image.shape + ): # i.e. if the size has changed. + self.layer_0_image_actor.VisibilityOn() + self.rgb_image_extent = ( + 0, + input_image.shape[1] - 1, + 0, + input_image.shape[0] - 1, + 0, + input_image.shape[2] - 1, + ) + self.rgb_image_importer.SetDataExtent(self.rgb_image_extent) + self.rgb_image_importer.SetWholeExtent(self.rgb_image_extent) + + if ( + self.video_in_layer_2 and self.rgb_input.shape != input_image.shape + ): # i.e. if the size has changed. + self.layer_2_image_actor.VisibilityOn() + self.rgba_image_extent = ( + 0, + input_image.shape[1] - 1, + 0, + input_image.shape[0] - 1, + 0, + input_image.shape[2], + ) + self.rgba_image_importer.SetDataExtent(self.rgba_image_extent) + self.rgba_image_importer.SetWholeExtent(self.rgba_image_extent) + + if self.video_in_layer_0 or self.video_in_layer_2: + self.__update_video_image_cameras() + self.__update_projection_matrices() + + if self.video_in_layer_0: + self.rgb_input = input_image + self.rgb_frame = np.copy(self.rgb_input[:, :, ::-1]) + self.rgb_image_importer.SetImportVoidPointer(self.rgb_frame.data) + self.rgb_image_importer.SetDataExtent(self.rgb_image_extent) + self.rgb_image_importer.SetWholeExtent(self.rgb_image_extent) + self.rgb_image_importer.Modified() + self.rgb_image_importer.Update() + + if self.video_in_layer_2: + self.rgb_input = input_image + self.rgba_frame = 255 * np.ones( + ( + input_image.shape[0], + input_image.shape[1], + input_image.shape[2] + 1, + ), + dtype=np.uint8, + ) + self.rgba_frame[:, :, 0:3] = self.rgb_input + if self.mask_image is not None: + self.rgba_frame[:, :, 3:4] = self.mask_image + self.rgba_image_importer.SetImportVoidPointer(self.rgba_frame.data) + self.rgba_image_importer.SetDataExtent(self.rgba_image_extent) + self.rgba_image_importer.SetWholeExtent(self.rgba_image_extent) + self.rgba_image_importer.Modified() + self.rgba_image_importer.Update() + + def __update_video_image_camera(self, camera, image_extent): + """ + Internal method to position a renderers camera to face a video image, + and to maximise the view of the image in the viewport. """ - self.background_camera = self.background_renderer.GetActiveCamera() - origin = (0, 0, 0) spacing = (1, 1, 1) # Works out the number of millimetres to the centre of the image. - x_c = origin[0] + 0.5 * (self.image_extent[0] + - self.image_extent[1]) * spacing[0] - y_c = origin[1] + 0.5 * (self.image_extent[2] + - self.image_extent[3]) * spacing[1] + x_c = origin[0] + 0.5 * (image_extent[0] + image_extent[1]) * spacing[0] + y_c = origin[1] + 0.5 * (image_extent[2] + image_extent[3]) * spacing[1] # Works out the total size of the image in millimetres. - i_w = (self.image_extent[1] - self.image_extent[0] + 1) * spacing[0] - i_h = (self.image_extent[3] - self.image_extent[2] + 1) * spacing[1] + i_w = (image_extent[1] - image_extent[0] + 1) * spacing[0] + i_h = (image_extent[3] - image_extent[2] + 1) * spacing[1] # Works out the ratio of required size to actual size. w_r = i_w / self.width() @@ -246,14 +379,28 @@ def __update_video_image_camera(self): else: scale = 0.5 * i_h - self.background_camera.SetFocalPoint(x_c, y_c, 0.0) - self.background_camera.SetPosition(x_c, y_c, -1000) - self.background_camera.SetViewUp(0.0, -1.0, 0.0) - self.background_camera.SetClippingRange(990, 1010) - self.background_camera.SetParallelProjection(True) - self.background_camera.SetParallelScale(scale) + camera.SetFocalPoint(x_c, y_c, 0.0) + camera.SetPosition(x_c, y_c, -1000) + camera.SetViewUp(0.0, -1.0, 0.0) + camera.SetClippingRange(990, 1010) + camera.SetParallelProjection(True) + camera.SetParallelScale(scale) - def __update_projection_matrix(self): + def __update_video_image_cameras(self): + """ + Position the background renderer camera, so that the video image + is maximised and centralised in the screen. + """ + if self.video_in_layer_0: + self.__update_video_image_camera( + self.layer_0_renderer.GetActiveCamera(), self.rgb_image_extent + ) + if self.video_in_layer_2: + self.__update_video_image_camera( + self.layer_2_renderer.GetActiveCamera(), self.rgba_image_extent + ) + + def __update_projection_matrix(self, renderer, camera, input_image): """ If a camera_matrix is available, then we are using a calibrated camera. This method recomputes the projection matrix, dependent on window size. @@ -263,48 +410,60 @@ def __update_projection_matrix(self): if self.camera_matrix is not None: - if self.input is None: - raise ValueError('Camera matrix is provided, but no image.') - - vtk_ren = self.get_foreground_renderer() - vtk_cam = self.get_foreground_camera() - - opengl_mat, vtk_mat = \ - cm.set_camera_intrinsics(vtk_ren, - vtk_cam, - self.input.shape[1], - self.input.shape[0], - self.camera_matrix[0][0], - self.camera_matrix[1][1], - self.camera_matrix[0][2], - self.camera_matrix[1][2], - self.clipping_range[0], - self.clipping_range[1] - ) - - vpx, vpy, vpw, vph = cm.compute_scissor(self.width(), - self.height(), - self.input.shape[1], - self.input.shape[0], - self.aspect_ratio - ) - - x_min, y_min, x_max, y_max = cm.compute_viewport(self.width(), - self.height(), - vpx, - vpy, - vpw, - vph - ) - - self.get_foreground_renderer().SetViewport(x_min, - y_min, - x_max, - y_max) + if input_image is None: + raise ValueError("Camera matrix is provided, but no image.") + + opengl_mat, vtk_mat = cm.set_camera_intrinsics( + renderer, + camera, + input_image.shape[1], + input_image.shape[0], + self.camera_matrix[0][0], + self.camera_matrix[1][1], + self.camera_matrix[0][2], + self.camera_matrix[1][2], + self.clipping_range[0], + self.clipping_range[1], + ) + + vpx, vpy, vpw, vph = cm.compute_scissor( + self.width(), + self.height(), + input_image.shape[1], + input_image.shape[0], + self.aspect_ratio, + ) + + x_min, y_min, x_max, y_max = cm.compute_viewport( + self.width(), self.height(), vpx, vpy, vpw, vph + ) + + renderer.SetViewport(x_min, y_min, x_max, y_max) vtk_rect = vtk.vtkRecti(vpx, vpy, vpw, vph) - vtk_cam.SetUseScissor(True) - vtk_cam.SetScissorRect(vtk_rect) + camera.SetUseScissor(True) + camera.SetScissorRect(vtk_rect) + + return opengl_mat, vtk_mat + + def __update_projection_matrices(self): + """ + If a camera_matrix is available, then we are using a calibrated camera. + This method recomputes the projection matrix, dependent on window size. + """ + renderer = self.get_foreground_renderer(layer=1) + opengl_mat, vtk_mat = self.__update_projection_matrix( + renderer, + renderer.GetActiveCamera(), + self.rgb_input, + ) + + renderer = self.get_foreground_renderer(layer=3) + opengl_mat, vtk_mat = self.__update_projection_matrix( + renderer, + renderer.GetActiveCamera(), + self.rgb_input, + ) return opengl_mat, vtk_mat @@ -318,8 +477,8 @@ def resizeEvent(self, ev): :param ev: Event """ super(VTKOverlayWindow, self).resizeEvent(ev) - self.__update_video_image_camera() - self.__update_projection_matrix() + self.__update_video_image_cameras() + self.__update_projection_matrices() self.Render() def set_camera_matrix(self, camera_matrix): @@ -329,7 +488,7 @@ def set_camera_matrix(self, camera_matrix): """ vm.validate_camera_matrix(camera_matrix) self.camera_matrix = camera_matrix - opengl_mat, vtk_mat = self.__update_projection_matrix() + opengl_mat, vtk_mat = self.__update_projection_matrices() self.Render() return opengl_mat, vtk_mat @@ -340,9 +499,13 @@ def set_camera_pose(self, camera_to_world): """ vm.validate_rigid_matrix(camera_to_world) self.camera_to_world = camera_to_world - vtk_cam = self.get_foreground_camera() vtk_mat = mu.create_vtk_matrix_from_numpy(camera_to_world) - cm.set_camera_pose(vtk_cam, vtk_mat, self.opencv_style) + cm.set_camera_pose( + self.layer_1_renderer.GetActiveCamera(), vtk_mat, self.opencv_style + ) + cm.set_camera_pose( + self.layer_3_renderer.GetActiveCamera(), vtk_mat, self.opencv_style + ) self.Render() def add_vtk_models(self, models, layer=1): @@ -356,13 +519,19 @@ def add_vtk_models(self, models, layer=1): """ if layer == 0: - raise ValueError("You shouldn't add actors to the backgroud scene") + raise ValueError("You shouldn't add actors to the background video.") if layer == 1: - renderer = self.foreground_renderer + renderer = self.layer_1_renderer elif layer == 2: - renderer = self.generic_overlay_renderer + raise ValueError("You shouldn't add actors to the midground video.") + + elif layer == 3: + renderer = self.layer_3_renderer + + elif layer == 4: + renderer = self.layer_4_renderer else: raise ValueError("Invalid layer specified") @@ -370,8 +539,7 @@ def add_vtk_models(self, models, layer=1): for model in models: renderer.AddActor(model.actor) if model.get_outline(): - renderer.AddActor( - model.get_outline_actor(renderer.GetActiveCamera())) + renderer.AddActor(model.get_outline_actor(renderer.GetActiveCamera())) if self.reset_camera: renderer.ResetCamera() @@ -385,13 +553,19 @@ def add_vtk_actor(self, actor, layer=1): """ if layer == 0: - raise ValueError("You shouldn't add actors to the background scene") + raise ValueError("You shouldn't add actors to the background video.") if layer == 1: - renderer = self.foreground_renderer + renderer = self.layer_1_renderer elif layer == 2: - renderer = self.generic_overlay_renderer + raise ValueError("You shouldn't add actors to the midground video.") + + elif layer == 3: + renderer = self.layer_3_renderer + + elif layer == 4: + renderer = self.layer_4_renderer else: raise ValueError("Invalid layer specified") @@ -401,27 +575,85 @@ def add_vtk_actor(self, actor, layer=1): if self.reset_camera: renderer.ResetCamera() - def get_foreground_renderer(self): + def get_background_image_actor(self, layer=0): + """ + Returns one of the background video layers, depending on + the constructor arguments. So, either layer 0 or 2. """ - Returns the foreground vtkRenderer. + if layer == 0: + return self.layer_0_image_actor + if layer == 1: + raise ValueError("Layer 1 is not a background renderer.") + if layer == 2: + return self.layer_2_image_actor + if layer == 3: + raise ValueError("Layer 3 is not a background renderer.") + if layer == 4: + raise ValueError("Layer 3 is not a background renderer.") + + raise ValueError("Didn't find background renderer.") + + def get_background_renderer(self, layer=0): + """ + Returns one of the background video layers, depending on + the constructor arguments. So, either layer 0 or 2. + """ + if layer == 0: + return self.layer_0_renderer + if layer == 1: + raise ValueError("Layer 1 is not a background renderer.") + if layer == 2: + return self.layer_2_renderer + if layer == 3: + raise ValueError("Layer 3 is not a background renderer.") + if layer == 4: + raise ValueError("Layer 3 is not a background renderer.") + + raise ValueError("Didn't find background renderer.") + + def get_foreground_renderer(self, layer=1): + """ + Returns the foreground vtkRenderer. For legacy compatibility, + this will assume layer 1, like this class was pre-Feb 3rd 2024. :return: vtkRenderer """ - return self.foreground_renderer + if layer == 0: + raise ValueError("Layer 0 is not a foreground renderer.") + if layer == 1: + return self.layer_1_renderer + if layer == 2: + raise ValueError("Layer 2 is not a foreground renderer.") + if layer == 3: + return self.layer_3_renderer + if layer == 4: + raise ValueError("Layer 4 is only for annotations like text.") + + raise ValueError(f"Invalid layer specification:{layer}") - def get_foreground_camera(self): + def get_foreground_camera(self, layer=1): """ - Returns the camera for the foreground renderer. + Returns the camera for the foreground vtkRenderer. For legacy compatibility, + this will assume layer 1, like this class was pre-Feb 3rd 2024. :returns: vtkCamera """ - return self.foreground_renderer.GetActiveCamera() + renderer = self.get_foreground_renderer(layer) + return renderer.GetActiveCamera() + + def set_foreground_camera(self, camera, layer=1): + """ + Set the foreground camera to track the view in another window. For legacy compatibility, + this will assume layer 1, like this class was pre-Feb 3rd 2024. + """ + renderer = self.get_foreground_renderer(layer) + renderer.SetActiveCamera(camera) - def set_foreground_camera(self, camera): + def get_overlay_renderer(self): """ - Set the foreground camera to track the view in another window. + This returns the top-most layer, where you might put text annotations for example. """ - self.foreground_renderer.SetActiveCamera(camera) + return self.layer_4_renderer def set_screen(self, screen): """ @@ -472,9 +704,9 @@ def convert_scene_to_numpy_array(self): self.vtk_array = self.vtk_image.GetPointData().GetScalars() number_of_components = self.vtk_array.GetNumberOfComponents() - np_array = vtk_to_numpy(self.vtk_array).reshape(height, - width, - number_of_components) + np_array = vtk_to_numpy(self.vtk_array).reshape( + height, width, number_of_components + ) self.output = cv2.flip(np_array, flipCode=0) return self.output @@ -489,20 +721,30 @@ def save_scene_to_file(self, file_name): self.output = cv2.cvtColor(self.output, cv2.COLOR_RGB2BGR) cv2.imwrite(file_name, self.output) - def get_camera_state(self): + def get_camera_state(self, layer=1): """ Get all the necessary variables to allow the camera - view to be restored. + view to be restored. For legacy compatibility, + this will assume layer 1, like this class was pre-Feb 3rd 2024. """ # pylint: disable=unused-variable, eval-used - camera = self.get_foreground_camera() + renderer = self.get_foreground_renderer(layer) + camera = renderer.GetActiveCamera() camera_properties = {} - properties_to_save = ["Position", "FocalPoint", "ViewUp", "ViewAngle", - "ParallelProjection", "ParallelScale", - "ClippingRange", "EyeAngle", "EyeSeparation", - "UseOffAxisProjection"] + properties_to_save = [ + "Position", + "FocalPoint", + "ViewUp", + "ViewAngle", + "ParallelProjection", + "ParallelScale", + "ClippingRange", + "EyeAngle", + "EyeSeparation", + "UseOffAxisProjection", + ] for camera_property in properties_to_save: # eval will run commands of the form @@ -513,13 +755,15 @@ def get_camera_state(self): return camera_properties - def set_camera_state(self, camera_properties): + def set_camera_state(self, camera_properties, layer=1): """ - Set the camera properties to a particular view poisition/angle etc. + Set the camera properties to a particular view poisition/angle etc. For legacy compatibility, + this will assume layer 1, like this class was pre-Feb 3rd 2024. """ # pylint: disable=unused-variable, eval-used - camera = self.get_foreground_camera() + renderer = self.get_foreground_renderer(layer) + camera = renderer.GetActiveCamera() for camera_property, value in camera_properties.items(): # eval statements 'camera.SetPosition(position)', diff --git a/tests/camera/test_liver_overlay.py b/tests/camera/test_liver_overlay.py index 4dcfddea..cf9e7618 100644 --- a/tests/camera/test_liver_overlay.py +++ b/tests/camera/test_liver_overlay.py @@ -13,12 +13,15 @@ skip_pytest_in_runner_macos = pytest.mark.skipif( platform.system() == "Darwin", - reason=f'for [{platform.system()} OSs with CI=[{os.environ.get("CI")}] with RUNNER_OS=[{os.environ.get("RUNNER_OS")}] ' - f'{os.environ.get("SESSION_MANAGER")[0:20] if (platform.system() == "Darwin" and os.environ.get("GITHUB_ACTIONS") == None) else ""} ' - f'with {os.environ.get("XDG_CURRENT_DESKTOP") if (platform.system() == "Darwin" and os.environ.get("GITHUB_ACTIONS") == None) else ""} ' + reason=f'for [{platform.system()} OSs with ' + f'CI=[{os.environ.get("CI")}] with ' + f'RUNNER_OS=[{os.environ.get("RUNNER_OS")}] ' + f'SESSION_MANAGER=[{os.environ.get("SESSION_MANAGER")[0:20] if (platform.system() == "Darwin" and os.environ.get("GITHUB_ACTIONS") is not None and os.environ.get("SESSION_MANAGER") is not None) else ""}] ' + f'XDG_CURRENT_DESKTOP=[{os.environ.get("XDG_CURRENT_DESKTOP") if (platform.system() == "Darwin" and os.environ.get("GITHUB_ACTIONS") is not None) else ""}] ' f'due to issues with Fatal Python error: Segmentation fault' ) + def _reproject_and_save_image(image, model_to_camera, point_cloud, @@ -59,6 +62,7 @@ def _reproject_and_save_image(image, cv2.imwrite(output_file, output_image) + @skip_pytest_in_runner_macos def test_overlay_liver_points(setup_vtk_overlay_window): """ diff --git a/tests/conftest.py b/tests/conftest.py index 9a6efc01..68d87a3d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -21,7 +21,6 @@ def setup_qt(): @pytest.fixture(scope="session") def setup_vtk_err(setup_qt): """ Used to send VTK errors to file instead of screen. """ - err_out = vtk.vtkFileOutputWindow() err_out.SetFileName('tests/output/vtk.err.txt') vtk_std_err = vtk.vtkOutputWindow() @@ -32,11 +31,9 @@ def setup_vtk_err(setup_qt): @pytest.fixture(scope="function") def setup_vtk_overlay_window(setup_vtk_err): """ - This function so you can select offscreen or not, while debugging. - - `init_widget_flag` is set to false `init_widget_flag` when testing in Linux OS machines. - Otherwise, `init_widget_flag = True`, calling `self.Initialize` and `self.Start` in the init function of - VTKOverlayWindow class + Sets `init_widget_flag` to False on Linux, and True on Windows and Mac. + When init_widget_flag==True, the VTKOverlayWindow constructor calls + `self.Initialize` and `self.Start` when creating the widget. """ vtk_std_err, setup_qt = setup_vtk_err @@ -52,7 +49,10 @@ def setup_vtk_overlay_window(setup_vtk_err): @pytest.fixture(scope="function") def setup_vtk_overlay_window_no_init(setup_vtk_err): - """ This function so you can select offscreen or not, while debugging. """ + """ + Similar to the above function, except init_widget=False, so + `self.Initialize` and `self.Start` are not called in the VTKOverlayWindow constructor. + """ vtk_std_err, setup_qt = setup_vtk_err @@ -60,14 +60,10 @@ def setup_vtk_overlay_window_no_init(setup_vtk_err): return vtk_overlay, vtk_std_err, setup_qt -# Note: These windows will persist while all unit tests run. -# Don't waste time trying to debug why you see >1 windows. -@pytest.fixture(scope="function") -def vtk_overlay_with_gradient_image(setup_vtk_overlay_window): - """ Creates a VTKOverlayWindow with gradient image. """ - - vtk_overlay, vtk_std_err, setup_qt = setup_vtk_overlay_window - +def _create_gradient_image(): + """ + Creates a dummy gradient image for testing only. + """ width = 512 height = 256 image = np.ones((height, width, 3), dtype=np.uint8) @@ -76,6 +72,17 @@ def vtk_overlay_with_gradient_image(setup_vtk_overlay_window): image[y][x][0] = y image[y][x][1] = y image[y][x][2] = y + return image + + +# Note: These windows will persist while all unit tests run. +# Don't waste time trying to debug why you see >1 windows. +@pytest.fixture(scope="function") +def vtk_overlay_with_gradient_image(setup_vtk_overlay_window): + """ Creates a VTKOverlayWindow with gradient image. """ + + vtk_overlay, vtk_std_err, setup_qt = setup_vtk_overlay_window + image = _create_gradient_image() vtk_overlay.set_video_image(image) return image, vtk_overlay, vtk_std_err, setup_qt @@ -93,3 +100,59 @@ def vtk_interlaced_stereo_window(setup_vtk_err): vtk_interlaced = VTKStereoInterlacedWindow(offscreen=False, init_widget=init_widget_flag) return vtk_interlaced, vtk_std_err, setup_qt + + +@pytest.fixture(scope="function") +def setup_vtk_overlay_window_video_only_layer_2(setup_vtk_err): + """ + Sets `init_widget_flag` to False on Linux, and True on Windows and Mac. + When init_widget_flag==True, the VTKOverlayWindow constructor calls + `self.Initialize` and `self.Start` when creating the widget. + + As of Issue #222: And also sets video_in_layer_0=False and video_in_layer_2=True + in VTKOverlayWindow constructor. See VTKOverlayWindow docstring for explanation. + """ + + vtk_std_err, setup_qt = setup_vtk_err + + if platform.system() == 'Linux': + init_widget_flag = False + else: + init_widget_flag = True + + vtk_overlay = VTKOverlayWindow(offscreen=False, + init_widget=init_widget_flag, + video_in_layer_0=False, + video_in_layer_2=True + ) + image = _create_gradient_image() + vtk_overlay.set_video_image(image) + return vtk_overlay, vtk_std_err, setup_qt + + +@pytest.fixture(scope="function") +def setup_vtk_overlay_window_video_both_layer_0_and_2(setup_vtk_err): + """ + Sets `init_widget_flag` to False on Linux, and True on Windows and Mac. + When init_widget_flag==True, the VTKOverlayWindow constructor calls + `self.Initialize` and `self.Start` when creating the widget. + + As of Issue #222: And also sets video_in_layer_0=True and video_in_layer_2=True + in VTKOverlayWindow constructor. See VTKOverlayWindow docstring for explanation. + """ + + vtk_std_err, setup_qt = setup_vtk_err + + if platform.system() == 'Linux': + init_widget_flag = False + else: + init_widget_flag = True + + vtk_overlay = VTKOverlayWindow(offscreen=False, + init_widget=init_widget_flag, + video_in_layer_0=True, + video_in_layer_2=True + ) + image = _create_gradient_image() + vtk_overlay.set_video_image(image) + return vtk_overlay, vtk_std_err, setup_qt diff --git a/tests/models/test_vtk_cylinder_model.py b/tests/models/test_vtk_cylinder_model.py index 995e4bf5..79f4c4bd 100644 --- a/tests/models/test_vtk_cylinder_model.py +++ b/tests/models/test_vtk_cylinder_model.py @@ -50,6 +50,6 @@ def test_cylinder_model(setup_vtk_overlay_window): # You don't really want this in a unit test, otherwise you can't exit. # If you want to do interactive testing, please uncomment the following line - # _pyside_qt_app.exec() + #_pyside_qt_app.exec() widget.close() diff --git a/tests/models/test_vtk_surface_model.py b/tests/models/test_vtk_surface_model.py index b871e5fe..3f8c9160 100644 --- a/tests/models/test_vtk_surface_model.py +++ b/tests/models/test_vtk_surface_model.py @@ -186,10 +186,10 @@ def test_flat_shaded_on_coloured_background(setup_vtk_overlay_window): widget, _, app = setup_vtk_overlay_window widget.add_vtk_actor(model.actor) model.set_no_shading(True) - widget.background_renderer.SetBackground(0, 0, 1) + widget.get_background_renderer().SetBackground(0, 0, 1) widget.show() model.set_no_shading(False) - widget.background_renderer.SetBackground(0, 1, 0) + widget.get_background_renderer().SetBackground(0, 1, 0) widget.show() # app.exec() diff --git a/tests/widgets/test_vtk_overlay_window.py b/tests/widgets/test_vtk_overlay_window.py index d6d3d291..4fcca704 100644 --- a/tests/widgets/test_vtk_overlay_window.py +++ b/tests/widgets/test_vtk_overlay_window.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import numpy as np -import platform import pytest from vtkmodules.vtkCommonColor import vtkNamedColors from vtkmodules.vtkFiltersSources import vtkConeSource @@ -12,8 +11,6 @@ ) from PySide6.QtWidgets import QApplication, QWidget, QVBoxLayout from vtkmodules.qt.QVTKRenderWindowInteractor import QVTKRenderWindowInteractor -from sksurgeryvtk.widgets.vtk_overlay_window import VTKOverlayWindow - import sksurgeryvtk.models.vtk_point_model as pm import sksurgeryvtk.models.vtk_surface_model as sm @@ -23,7 +20,7 @@ def test_vtk_render_window_settings(setup_vtk_overlay_window): assert not widget.GetRenderWindow().GetStereoRender() assert not widget.GetRenderWindow().GetStereoCapableWindow() - # assert widget.GetRenderWindow().GetAlphaBitPlanes() + assert widget.GetRenderWindow().GetAlphaBitPlanes() assert widget.GetRenderWindow().GetMultiSamples() == 0 widget.close() @@ -33,7 +30,7 @@ def test_vtk_render_window_settings_no_init(setup_vtk_overlay_window_no_init): assert not widget.GetRenderWindow().GetStereoRender() assert not widget.GetRenderWindow().GetStereoCapableWindow() - # assert widget.GetRenderWindow().GetAlphaBitPlanes() + assert widget.GetRenderWindow().GetAlphaBitPlanes() assert widget.GetRenderWindow().GetMultiSamples() == 0 widget.close() @@ -41,28 +38,30 @@ def test_vtk_render_window_settings_no_init(setup_vtk_overlay_window_no_init): def test_vtk_foreground_render_settings(setup_vtk_overlay_window): widget, _vtk_std_err, _pyside_qt_app = setup_vtk_overlay_window - assert widget.foreground_renderer.GetLayer() == 1 - assert widget.foreground_renderer.GetUseDepthPeeling() + layer = widget.get_foreground_renderer().GetLayer() + assert widget.get_foreground_renderer().GetLayer() == 1 + assert widget.get_foreground_renderer().GetUseDepthPeeling() widget.close() def test_vtk_background_render_settings(setup_vtk_overlay_window): widget, _vtk_std_err, _pyside_qt_app = setup_vtk_overlay_window - assert widget.background_renderer.GetLayer() == 0 - assert not widget.background_renderer.GetInteractive() + assert widget.get_background_renderer().GetLayer() == 0 + assert not widget.get_background_renderer().GetInteractive() widget.close() def test_image_importer(setup_vtk_overlay_window): widget, _vtk_std_err, _pyside_qt_app = setup_vtk_overlay_window - width, height, _number_of_scalar_components = widget.input.shape - expected_extent = (0, height - 1, 0, width - 1, 0, 0) + width, height, _number_of_scalar_components = widget.rgb_input.shape + expected_extent = (0, height - 1, 0, width - 1, 0, 2) + actual_extent = widget.rgb_image_importer.GetDataExtent() - assert widget.image_importer.GetDataExtent() == expected_extent - assert widget.image_importer.GetDataScalarTypeAsString() == "unsigned char" - assert widget.image_importer.GetNumberOfScalarComponents() == 3 + assert actual_extent == expected_extent + assert widget.rgb_image_importer.GetDataScalarTypeAsString() == "unsigned char" + assert widget.rgb_image_importer.GetNumberOfScalarComponents() == 3 widget.close() @@ -116,22 +115,21 @@ def test_basic_pyside_vtk_pipeline(): ren = vtkRenderer() ren.AddActor(coneActor) - qvtk_render_window_iterator = QVTKRenderWindowInteractor() - qvtk_render_window_iterator.GetRenderWindow().AddRenderer(ren) - qvtk_render_window_iterator.resize(100, 100) + qvtk_render_window_interactor = QVTKRenderWindowInteractor() + qvtk_render_window_interactor.GetRenderWindow().AddRenderer(ren) + qvtk_render_window_interactor.resize(100, 100) - layout.addWidget(qvtk_render_window_iterator) + layout.addWidget(qvtk_render_window_interactor) # To exit window using 'q' or 'e' key - qvtk_render_window_iterator.AddObserver("ExitEvent", lambda o, e, a=_pyside_qt_app: a.quit()) - - qvtk_render_window_iterator.Initialize() - qvtk_render_window_iterator.Start() + qvtk_render_window_interactor.AddObserver("ExitEvent", lambda o, e, a=_pyside_qt_app: a.quit()) + qvtk_render_window_interactor.Initialize() + qvtk_render_window_interactor.Start() # You don't really want this in a unit test, otherwise you can't exit. # If you want to do interactive testing, please uncomment the following line # _pyside_qt_app.exec() - qvtk_render_window_iterator.close() + qvtk_render_window_interactor.close() def test_basic_cone_overlay(vtk_overlay_with_gradient_image): @@ -227,6 +225,10 @@ def test_add_model_to_background_renderer_raises_error(vtk_overlay_with_gradient with pytest.raises(ValueError): widget.add_vtk_models(surface, layer=0) + + with pytest.raises(ValueError): + widget.add_vtk_models(surface, layer=2) + widget.close() @@ -238,17 +240,17 @@ def test_add_models_to_foreground_renderer(vtk_overlay_with_gradient_image): # If no layer is specified, default is 0 widget.add_vtk_models(liver) - foreground_actors = widget.foreground_renderer.GetActors() + foreground_actors = widget.get_foreground_renderer().GetActors() assert foreground_actors.GetNumberOfItems() == 1 # Explicitly specify use of foreground renderer widget.add_vtk_models(tumors, 1) - foreground_actors = widget.foreground_renderer.GetActors() + foreground_actors = widget.get_foreground_renderer().GetActors() assert foreground_actors.GetNumberOfItems() == 2 # Check overlay renderer is empty - overlay_renderer_actors = widget.generic_overlay_renderer.GetActors() + overlay_renderer_actors = widget.get_overlay_renderer().GetActors() assert overlay_renderer_actors.GetNumberOfItems() == 0 widget.close() @@ -258,17 +260,17 @@ def test_add_models_to_overlay_renderer(vtk_overlay_with_gradient_image): tumors = [sm.VTKSurfaceModel('tests/data/models/Liver/liver_tumours.vtk', (1.0, 1.0, 1.0))] _image, widget, _vtk_std_err, _pyside_qt_app = vtk_overlay_with_gradient_image - widget.add_vtk_models(liver, 2) + widget.add_vtk_models(liver, 4) - overlay_actors = widget.generic_overlay_renderer.GetActors() + overlay_actors = widget.get_overlay_renderer().GetActors() assert overlay_actors.GetNumberOfItems() == 1 - widget.add_vtk_models(tumors, 2) + widget.add_vtk_models(tumors, 4) - overlay_actors = widget.generic_overlay_renderer.GetActors() + overlay_actors = widget.get_overlay_renderer().GetActors() assert overlay_actors.GetNumberOfItems() == 2 # Check foreground is empty - foreground_actors = widget.foreground_renderer.GetActors() + foreground_actors = widget.get_foreground_renderer().GetActors() assert foreground_actors.GetNumberOfItems() == 0 widget.close() diff --git a/tests/widgets/test_vtk_overlay_window_5_layers.py b/tests/widgets/test_vtk_overlay_window_5_layers.py new file mode 100644 index 00000000..d625b942 --- /dev/null +++ b/tests/widgets/test_vtk_overlay_window_5_layers.py @@ -0,0 +1,243 @@ +# -*- coding: utf-8 -*- + +import numpy as np +import pytest +import sksurgeryvtk.models.vtk_surface_model as sm + + +def test_overlay_window_video_0(vtk_overlay_with_gradient_image): + """ + As of Issue #222: Tests the 'default' or 'legacy' configuration. + + Prior to Issue #222, there were 3 layers: + + - Layer 0: Video + - Layer 1: VTK models, overlaid on Video + - Layer 2: Text annotations. + + So, if you changed the opacity of the VTK models, you got naive alpha blending for Augmented Reality (AR). + + For Issue #222, there are now 5 rendering layers. So, by default you should get: + + - widget.get_background_renderer() should return widget.layer_0_renderer + - widget.get_foreground_renderer() should return widget.layer_1_renderer + - widget.get_overlay_renderer() should return widget.layer_4_renderer + + and these 3 are the equivalent of the 3 layers that existed prior to Issue #222. + """ + liver = sm.VTKSurfaceModel('tests/data/models/Liver/liver.vtk', (1.0, 0.0, 0.0), opacity=0.2) + tumors = sm.VTKSurfaceModel('tests/data/models/Liver/liver_tumours.vtk', (0.0, 1.0, 0.0), opacity=0.2) + image, vtk_overlay, vtk_std_err, app = vtk_overlay_with_gradient_image + vtk_overlay.add_vtk_models([liver, tumors]) + + vtk_overlay.resize(512, 256) + vtk_overlay.show() + vtk_overlay.Render() + + bg_ren = vtk_overlay.get_background_renderer() + assert bg_ren == vtk_overlay.layer_0_renderer + + fg_ren = vtk_overlay.get_foreground_renderer() + assert fg_ren == vtk_overlay.layer_1_renderer + + ov_ren = vtk_overlay.get_overlay_renderer() + assert ov_ren == vtk_overlay.layer_4_renderer + + foreground_actors = fg_ren.GetActors() + assert foreground_actors.GetNumberOfItems() == 2 + + # You don't really want this in a unit test, :-) + # otherwise you can't exit. It's kept here for interactive testing. + #app.exec() + + +def _create_gradient_image_2(): + """ + Creates a dummy gradient image for testing only. + """ + width = 512 + height = 256 + image = np.ones((height, width, 3), dtype=np.uint8) + for y in range(height): + for x in range(width): + image[y][x][0] = y + image[y][x][1] = y + image[y][x][2] = y + return image + + +def _create_rgba_alpha_mask(): + """ + Creates a dummy mask image for testing only. + """ + width = 512 + height = 256 + cx = width / 2 + cy = height / 2 + image = 255 * np.ones((height, width, 1), dtype=np.uint8) + for y in range(height): + for x in range(width): + dist = np.sqrt((y-cy) * (y-cy) + (x-cx) * (x-cx)) + if dist < 40: + image[y][x][0] = 0 + return image + + +def test_overlay_window_video_2(setup_vtk_overlay_window_video_only_layer_2): + """ + As of Issue #222: Tests the first new configuration. + + The background video can be in layer 0, 2, or both, for different visual effects. + + So, with VTKOverlayWindow(video_in_layer_0=False, video_in_layer_2=True) you should have: + + - widget.get_background_renderer() should return widget.layer_2_renderer + - widget.get_foreground_renderer() should return widget.layer_1_renderer + - widget.get_overlay_renderer() should return widget.layer_4_renderer + + This will render the video in layer 2, i.e. IN FRONT of the models in layer 1. + The idea then is that you can set a mask on the video image to use the alpha channel + to set some of the video pixels to be transparent, so you can 'see through' the video. + This gives the illusion of peeking through the video, and seeing the models behind + the video. The mask could also be faded, so you don't get quite such a hard edge. + """ + liver = sm.VTKSurfaceModel('tests/data/models/Liver/liver.vtk', (1.0, 0.0, 0.0), opacity=0.2) + tumors = sm.VTKSurfaceModel('tests/data/models/Liver/liver_tumours.vtk', (0.0, 1.0, 0.0), opacity=0.2) + vtk_overlay, vtk_std_err, app = setup_vtk_overlay_window_video_only_layer_2 + + mask = _create_rgba_alpha_mask() + vtk_overlay.set_video_mask(mask) + image = _create_gradient_image_2() + vtk_overlay.set_video_image(image) + vtk_overlay.add_vtk_models([liver, tumors]) + vtk_overlay.resize(512, 256) + vtk_overlay.show() + vtk_overlay.Render() + + bg_ren = vtk_overlay.get_background_renderer() + assert bg_ren == vtk_overlay.layer_2_renderer + + fg_ren = vtk_overlay.get_foreground_renderer() + assert fg_ren == vtk_overlay.layer_1_renderer + + ov_ren = vtk_overlay.get_overlay_renderer() + assert ov_ren == vtk_overlay.layer_4_renderer + + foreground_actors = fg_ren.GetActors() + assert foreground_actors.GetNumberOfItems() == 2 + + # You don't really want this in a unit test, :-) + # otherwise you can't exit. It's kept here for interactive testing. + #app.exec() + + +def test_overlay_window_video_both(setup_vtk_overlay_window_video_both_layer_0_and_2): + """ + As of Issue #222: Tests the second new configuration. + + The background video can be in layer 0, 2, or both, for different visual effects. + + So, with VTKOverlayWindow(video_in_layer_0=True, video_in_layer_2=True) you should have: + + - widget.get_background_renderer() should return widget.layer_0_renderer containing the video. + - widget.get_foreground_renderer() should return widget.layer_1_renderer + - widget.get_overlay_renderer() should return widget.layer_4_renderer + + So, in this test, we are using both video layers and testing putting models in layer 1, + which is the default layer. So the models appear behind the masked video, but the video + appears in both layers 0 and 2, so the background behind the models also shows the video. + + In addition, you can retrieve the foreground renderer, specifying the layer, either 1 or 3. + """ + liver = sm.VTKSurfaceModel('tests/data/models/Liver/liver.vtk', (1.0, 0.0, 0.0), opacity=0.2) + tumors = sm.VTKSurfaceModel('tests/data/models/Liver/liver_tumours.vtk', (0.0, 1.0, 0.0), opacity=0.2) + vtk_overlay, vtk_std_err, app = setup_vtk_overlay_window_video_both_layer_0_and_2 + + mask = _create_rgba_alpha_mask() + vtk_overlay.set_video_mask(mask) + image = _create_gradient_image_2() + vtk_overlay.set_video_image(image) + vtk_overlay.add_vtk_models([liver, tumors]) + vtk_overlay.resize(512, 256) + vtk_overlay.show() + vtk_overlay.Render() + + bg_ren = vtk_overlay.get_background_renderer() + assert bg_ren == vtk_overlay.layer_0_renderer + + fg_ren = vtk_overlay.get_foreground_renderer() + assert fg_ren == vtk_overlay.layer_1_renderer + + ov_ren = vtk_overlay.get_overlay_renderer() + assert ov_ren == vtk_overlay.layer_4_renderer + + foreground_actors = fg_ren.GetActors() + assert foreground_actors.GetNumberOfItems() == 2 + + fg_ren = vtk_overlay.get_foreground_renderer(layer=1) + assert fg_ren == vtk_overlay.layer_1_renderer + + fg_ren = vtk_overlay.get_foreground_renderer(layer=3) + assert fg_ren == vtk_overlay.layer_3_renderer + + with pytest.raises(ValueError): + fg_ren = vtk_overlay.get_foreground_renderer(layer=0) + + with pytest.raises(ValueError): + fg_ren = vtk_overlay.get_foreground_renderer(layer=2) + + with pytest.raises(ValueError): + fg_ren = vtk_overlay.get_foreground_renderer(layer=4) + + with pytest.raises(ValueError): + fg_ren = vtk_overlay.get_foreground_renderer(layer=5) + + # You don't really want this in a unit test, :-) + # otherwise you can't exit. It's kept here for interactive testing. + #app.exec() + + +def test_overlay_window_combined_ar_look(setup_vtk_overlay_window_video_only_layer_2): + """ + As of Issue #222: Tests the third new configuration. + + Given all the options described above, in this test we aim for: + + - VTKOverlayWindow(video_in_layer_0=False, video_in_layer_2=True) to put video in layer 2. + - we apply a mask to the video to give the impression of seeing inside the model + - we add internal anatomy (e.g. tumour) behind the video in layer 2. + - we add external anatomy (e.g. liver) in front of the video in layer 2. + - we render the external anatomy as outline, and surface. + """ + liver = sm.VTKSurfaceModel('tests/data/models/Liver/liver.vtk', (1.0, 0.0, 0.0), opacity=0.2, outline=True) + tumors = sm.VTKSurfaceModel('tests/data/models/Liver/liver_tumours.vtk', (0.0, 1.0, 0.0), opacity=0.2) + vtk_overlay, vtk_std_err, app = setup_vtk_overlay_window_video_only_layer_2 + + mask = _create_rgba_alpha_mask() + vtk_overlay.set_video_mask(mask) + image = _create_gradient_image_2() + vtk_overlay.set_video_image(image) + vtk_overlay.add_vtk_models([tumors], layer=1) + vtk_overlay.add_vtk_models([liver], layer=3) + vtk_overlay.resize(512, 256) + vtk_overlay.show() + vtk_overlay.Render() + + bg_ren = vtk_overlay.get_background_renderer() + assert bg_ren == vtk_overlay.layer_2_renderer + + fg_ren = vtk_overlay.get_foreground_renderer(layer=1) + assert fg_ren == vtk_overlay.layer_1_renderer + + foreground_actors = fg_ren.GetActors() + assert foreground_actors.GetNumberOfItems() == 1 + + fg_ren = vtk_overlay.get_foreground_renderer(layer=3) + assert fg_ren == vtk_overlay.layer_3_renderer + + foreground_actors = fg_ren.GetActors() + assert foreground_actors.GetNumberOfItems() == 2 # i.e. surface AND outline. + + # You don't really want this in a unit test, :-) + # otherwise you can't exit. It's kept here for interactive testing. + #app.exec() \ No newline at end of file diff --git a/tests/widgets/test_vtk_overlay_window_with_outlines.py b/tests/widgets/test_vtk_overlay_window_with_outlines.py index ddd74b5e..464bae77 100644 --- a/tests/widgets/test_vtk_overlay_window_with_outlines.py +++ b/tests/widgets/test_vtk_overlay_window_with_outlines.py @@ -14,14 +14,14 @@ def test_surface_without_outline(vtk_overlay_with_gradient_image): opacity=0.1, outline=False)] widget.add_vtk_models(surface) outline_actor = surface[0].get_outline_actor( - widget.foreground_renderer.GetActiveCamera()) + widget.get_foreground_renderer().GetActiveCamera()) - foreground_actors = widget.foreground_renderer.GetActors() + foreground_actors = widget.get_foreground_renderer().GetActors() assert foreground_actors.GetNumberOfItems() == 1 widget.add_vtk_actor(outline_actor) - foreground_actors = widget.foreground_renderer.GetActors() + foreground_actors = widget.get_foreground_renderer().GetActors() assert foreground_actors.GetNumberOfItems() == 1 widget.resize(512, 256) @@ -44,7 +44,7 @@ def test_surface_outline_overlay(vtk_overlay_with_gradient_image): opacity=0.1, outline=True)] widget.add_vtk_models(surface) - foreground_actors = widget.foreground_renderer.GetActors() + foreground_actors = widget.get_foreground_renderer().GetActors() assert foreground_actors.GetNumberOfItems() == 2 widget.resize(512, 256)