Embedding Open3D in PySide or PyQt Application

[notaTeapot]

For a project of mine, I want to embed an o3d visualization into an existing python Qt application. I found this Stackoverflow page (https://stackoverflow.com/questions/58816766/how-to-embed-a-pptk-viewer-in-a-pyqt5-window) on how to integrate a pptk window into a pyqt5 app using the win32gui package. I adapted the code with Open3D and got it running inside a widget:

The code is pretty simple and you can interact with the scene just like with the standalone window:

from PySide2 import QtWidgets, QtGui,QtCore
import open3d as o3d
import win32gui
import sys

class MainWindow(QtWidgets.QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        widget = QtWidgets.QWidget()
        layout = QtWidgets.QGridLayout(widget)
        self.setCentralWidget(widget)

        pcd = o3d.io.read_point_cloud("example.pcd")
        self.vis = o3d.visualization.Visualizer()
        self.vis.create_window()
        self.vis.add_geometry(pcd)

        hwnd = win32gui.FindWindowEx(0, 0, None, "Open3D")
        self.window = QtGui.QWindow.fromWinId(hwnd)    
        self.windowcontainer = self.createWindowContainer(self.window, widget)
        layout.addWidget(self.windowcontainer, 0, 0)
        timer = QtCore.QTimer(self)
        timer.timeout.connect(self.update_vis)
        timer.start(1)

    def update_vis(self):
        #self.vis.update_geometry()
        self.vis.poll_events()
        self.vis.update_renderer()

if __name__ == '__main__':

    app = QtWidgets.QApplication(sys.argv)
    form = MainWindow()
    form.setWindowTitle('o3d Embed')
    form.setGeometry(100, 100, 600, 500)
    form.show()
    sys.exit(app.exec_())

[ssheorey]

Nice!

[JLUJYP]

I want to implement interaction of window, be about to change self.vis = o3d.visualization.Visualizer() to self.vis = o3d.visualization.VisualizerWithEditing() , but I didn't succeed, Do you know how to solve it, thank you very much!

[notaTeapot]

https://gist.github.com/notaTeapot/5feea53dccc921ea6af88a4d222e1e2b

[hustCYQ]

I found that many times after clicking other buttons of qt, the focus is not on vis. At this time, the keyboard input does not seem to be accepted by vis. How can I solve this problem?

[hustCYQ]

https://gist.github.com/notaTeapot/5feea53dccc921ea6af88a4d222e1e2b

I found that many times after clicking other buttons of qt, the focus is not on vis. At this time, the keyboard input does not seem to be accepted by vis. How can I solve this problem?

[elevenjiang1]

Not sure whether you can solve the problem about using o3d.visualization.VisualizerWithEditing()
I can not use @wpyq code, maybe it is because open3d will segment fault when selecting point with poll_events() and update_renderer().

Here is my code for just selecting points and embedding open3d into pyqt, in Ubuntu. So I also not use win32gui but use subprocess and 'wmctrl -l'. If you want to use it, please apt install wmctrl first.

import sys
import open3d as o3d
from PyQt5.QtWidgets import QApplication,QMainWindow,QWidget,QGridLayout

from PyQt5.QtGui import QWindow
from PyQt5.QtCore import QTimer,pyqtSignal

import example_ui
from subprocess import Popen, PIPE
from threading import Thread

class MainWindow(QMainWindow):
    begin_select_point_signal=pyqtSignal()
    def __init__(self,parent=None):
        super(MainWindow, self).__init__(parent)
        self.ui = example_ui.Ui_MainWindow()
        self.ui.setupUi(self)
        
        #1: Enable button
        self.ui.pushButton_helloworld.clicked.connect(self.helloworld_button_cb)
        self.ui.pushButton_show_points.clicked.connect(self.select_points_cb)
        
        #2: Start a Timer to update points
        self.o3d_selectpoint_vis_timer=QTimer(self)
        self.o3d_selectpoint_vis_timer.timeout.connect(self.vis_run_cb)
        
    
    def get_hwnd(self):
        hwnd = None
        while hwnd == None:
            proc = Popen('wmctrl -l', stdin=None, stdout=PIPE, stderr=None, shell=True)
            out, err = proc.communicate()
            for window in out.decode('utf-8').split('\n'):
                if 'Open3D' in window:
                    hwnd = int(window.split(' ')[0], 16)
                    print("Find hwnd is:",hwnd)
                    return hwnd    

    def helloworld_button_cb(self):
        print("You press Hello World")

    def select_points_cb(self):
        pcd = o3d.io.read_point_cloud("/home/elevenjiang/Documents/Project/VisionGuidance-Project/VisionGuidance/vision/data/Objects/wall.ply")
        
        #Must use no vis mode to embed into pyqt, but get win handle with vis mode first
        self.vis = o3d.visualization.VisualizerWithEditing()
        self.vis.create_window(visible=True)
        self.winid=self.get_hwnd()
        self.vis.destroy_window()
        self.vis = o3d.visualization.VisualizerWithEditing()
        self.vis.create_window(visible=False)
        
        #Embed window into widget
        self.sub_window = QWindow.fromWinId(self.winid)
        self.windowcontainer = QWidget.createWindowContainer(self.sub_window,self.ui.o3d_widget)
        layout=QGridLayout(self.ui.o3d_widget)
        layout.addWidget(self.windowcontainer,0,0)
        self.vis.add_geometry(pcd)
        
        #start a timmer in the main thread
        self.o3d_selectpoint_vis_timer.start(100)#Can not set too much otherwise will wait long time for first time
        
    def vis_run_cb(self):
        self.vis.run()
        self.vis.destroy_window()
        self.selected_points_list=self.vis.get_picked_points()
        
        self.o3d_selectpoint_vis_timer.stop()
        
        
if __name__ == '__main__':
    app = QApplication(sys.argv)
    window = MainWindow()
    window.show()
    sys.exit(app.exec_())

Thanks to everyone for discussing this problem here too much, meanwhile I am also curious about whether can we just move the open3d window into the middle of QT UI, but not embed it into QT? Maybe this will be much more convenient.

[rrmsm]

@elevenjiang1 Why does the program appear unresponsive after 5 seconds?

[ebhrz]

Since we don't have win32gui on linux, how can I use it on Linux?

[zapplelove]

It's hard to say. As I know, after starting the ros node, it seems open a new thread. Maybe you could try starting the ros node in init of open3d_display rather than in main function.

Your solution is very clever, thank you！

[zapplelove]

(https://stackoverflow.com/questions/58816766/how-to-embed-a-pptk-viewer-in-a-pyqt5-window)

here is the example demo on linux instead of using win32gui:

from PyQt5 import QtWidgets, QtGui,QtCore
import open3d as o3d
from subprocess import Popen, PIPE
# import win32gui
import sys

class MainWindow(QtWidgets.QMainWindow):
    def __init__(self):
        super(MainWindow, self).__init__()
        widget = QtWidgets.QWidget()
        layout = QtWidgets.QGridLayout(widget)
        self.setCentralWidget(widget)

        pcd = o3d.io.read_point_cloud("example.pcd")
        self.vis = o3d.visualization.Visualizer()
        self.vis.create_window()
        self.vis.add_geometry(pcd)

        # hwnd = win32gui.FindWindowEx(0, 0, None, "Open3D")
        hwnd = self.get_hwnd()
        self.window = QtGui.QWindow.fromWinId(hwnd)    
        self.windowcontainer = self.createWindowContainer(self.window, widget)
        layout.addWidget(self.windowcontainer, 0, 0)
        timer = QtCore.QTimer(self)
        timer.timeout.connect(self.update_vis)
        timer.start(1)

    def get_hwnd(self):
        hwnd = None
        while hwnd == None:
            proc = Popen('wmctrl -l', stdin=None, stdout=PIPE, stderr=None, shell=True)
            out, err = proc.communicate()
            for window in out.decode('utf-8').split('\n'):
                if 'Open3D' in window:
                    hwnd = int(window.split(' ')[0], 16)
                    return hwnd

    def update_vis(self):
        #self.vis.update_geometry()
        self.vis.poll_events()
        self.vis.update_renderer()

if __name__ == '__main__':

    app = QtWidgets.QApplication(sys.argv)
    form = MainWindow()
    form.setWindowTitle('o3d Embed')
    form.setGeometry(100, 100, 600, 500)
    form.show()
    sys.exit(app.exec_())

[HongruiTang]

I used pyqtgraph.opengl to show 3d pointcloud at last

Hi, can you share your code on linux or mac?

[WonjaeChoi0712]

Hello, I am looking for a solution for mac os(m2 chip).
I tried to get the hwnd through Quartz module but wasn't successful :(
Does anyone have any ideas on this issue??

[ebhrz]

Sorry for late reply, I just saw it. Here's my code

from PyQt5.QtWidgets import QProgressDialog, QFileDialog,QMainWindow,QApplication,QMessageBox,QDialog, QGraphicsScene, QGraphicsPixmapItem
from PyQt5.QtGui import QImage,QPixmap
import pyqtgraph as pg
import pyqtgraph.opengl as gl
from OpenGL.GL import GL_ONE, GL_BLEND
from OpenGL.GL.EXT.blend_minmax import GL_MAX_EXT
from ui.mainwin import Ui_MainWindow # Your layout

class MyApp(QMainWindow, Ui_MainWindow):
    def __init__(self):
        QMainWindow.__init__(self)
        Ui_MainWindow.__init__(self)
        # here to add the pyqtgraph opengl to a named widget defined in ui
        self.setupUi(self)
        self.glwidget = gl.GLViewWidget()
        g = gl.GLGridItem()
        g.scale(10, 10, 1)
        self.glwidget.addItem(g)
        # this is the name
        self.verticalLayout.addWidget(self.glwidget)

    def gen3d(self):
        # volume is a series of image here, just change to what you need
        tmp = np.zeros((*self.volume.shape,4))
        t1 = self.volume.copy()
        tmp[:,:,:,0]=t1+50
        tmp[:,:,:,1]=t1+50
        tmp[:,:,:,2]=t1+50
        tmp[:,:,:,3]=10
        tmp = np.transpose(tmp,[0,2,1,3])
        v = gl.GLVolumeItem(tmp)
        v.translate(-50,-256,-256)
        v.rotate(180,0,1,0)
        #v.setGLOptions({'glBlendFunc': (GL_ONE, GL_ONE), 'glBlendEquation': (GL_MAX_EXT, ), GL_BLEND: True})
        self.glwidget.clear()
        self.glwidget.addItem(v)
if __name__ == "__main__":
    app = QApplication(sys.argv)
    window = MyApp()
    window.show()
    sys.exit(app.exec_())

@HongruiTang
@WonjaeChoi0712 This works on my intel mac, I think it should also work on m2.