notebooks/flatlands.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Flatlands"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from PIL import Image\n",
    "import open3d as o3d\n",
    "import plotly.graph_objects as go\n",
    "import plotly.io as pio\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.figure_factory as ff"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## The point cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load point cloud data\n",
    "pcd = o3d.io.read_point_cloud(\"../data/happy_vrip_res3.ply\")\n",
    "\n",
    "# Scale for better labels\n",
    "points = np.asarray(pcd.points) * 100\n",
    "\n",
    "fig = go.Figure(go.Scatter3d(x=points[:, 0],\n",
    "                             y=points[:, 1],\n",
    "                             z=points[:, 2],\n",
    "                             mode=\"markers\",\n",
    "                             hovertemplate=\"<b>Point</b><br>x: %{x}<br>y: %{y}<br>z: %{z}<extra></extra>\",\n",
    "                             hoverlabel=dict(bgcolor=\"white\"),\n",
    "                             marker=dict(size=5,\n",
    "                                         color=points[:, 2],\n",
    "                                         colorscale='magma')))\n",
    "\n",
    "camera = dict(eye=dict(x=-1e-7, y=-1e-5, z=0.4))\n",
    "\n",
    "fig.update_layout(scene=dict(\n",
    "                    xaxis=dict(visible=False),\n",
    "                    yaxis=dict(visible=False),\n",
    "                    zaxis=dict(visible=False),\n",
    "                    aspectmode='data'),\n",
    "                  height=700,\n",
    "                  margin=dict(r=0, l=0, b=0, t=0, pad=0),\n",
    "                  scene_camera=camera,\n",
    "                  scene_dragmode=\"orbit\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save figure\n",
    "pio.write_html(fig,\n",
    "               file=f\"../_includes/figures/happy_buddha.html\",\n",
    "               full_html=False,\n",
    "               include_plotlyjs='cdn')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Image vs point cloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load and resize image\n",
    "size = 256, 256\n",
    "image_rgb = Image.open(\"../images/happy_buddha.jpg\")\n",
    "image_rgb.thumbnail(size)\n",
    "image_rgb = np.asarray(image_rgb)\n",
    "\n",
    "# Get size and color info\n",
    "height, width, channels = image_rgb.shape\n",
    "color = np.array([f\"rgb{rgb[0], rgb[1], rgb[2]}\" for rgb in image_rgb.reshape(-1, 3)])\n",
    "\n",
    "# Make grid to paint colors (or pixel values) on\n",
    "x, y = np.meshgrid(np.arange(width), np.arange(height))\n",
    "\n",
    "# Visualize image as pixel grid\n",
    "img_orig = go.Scattergl(x=x.reshape(-1),\n",
    "                        y=y.reshape(-1),\n",
    "                        mode='markers',\n",
    "                        marker=dict(size=6,\n",
    "                                    color=color,\n",
    "                                    symbol=\"square\"),\n",
    "                        hovertemplate=\"<b>Pixel:</b> col: %{x}, row: %{y}<extra></extra>\",\n",
    "                        hoverlabel=dict(bgcolor=color),\n",
    "                        showlegend=False)\n",
    "\n",
    "# Remove \"empty\" (black) pixels\n",
    "x_list, y_list, color_list = list(), list(), list()\n",
    "for i, rgb in enumerate(image_rgb.reshape(-1, 3)):\n",
    "    if sum(rgb) >= 10:\n",
    "        x_list.append(x.reshape(-1)[i])\n",
    "        y_list.append(y.reshape(-1)[i])\n",
    "        color_list.append(color[i])\n",
    "\n",
    "# Shuffle and scale to show that point clouds are unordered and not bound to a grid\n",
    "x = np.array(x_list) * 0.1\n",
    "y = np.array(y_list) * 0.1\n",
    "color = np.array(color_list)\n",
    "\n",
    "# Visualize image as 2D point cloud\n",
    "pcd_2d = go.Scattergl(x=x.reshape(-1),\n",
    "                      y=y.reshape(-1),\n",
    "                      mode='markers',\n",
    "                      marker=dict(size=6,\n",
    "                                  color=color,\n",
    "                                  symbol=\"circle\"),\n",
    "                      hovertemplate=\"<b>Point:</b> x: %{x}, y: %{y}<extra></extra>\",\n",
    "                      hoverlabel=dict(bgcolor=color),\n",
    "                      showlegend=False)\n",
    "\n",
    "fig = make_subplots(rows=1,\n",
    "                    cols=2,\n",
    "                    column_widths=[0.5, 0.5],\n",
    "                    horizontal_spacing=0.05,\n",
    "                    vertical_spacing=0,\n",
    "                    specs=[[dict(type=\"xy\"), dict(type=\"xy\")]])\n",
    "\n",
    "fig.add_trace(img_orig, row=1, col=1)\n",
    "fig.add_trace(pcd_2d, row=1, col=2)\n",
    "fig.update_layout(template=\"plotly_white\",\n",
    "                  yaxis=dict(tick0=0,\n",
    "                             range=(255, 0),\n",
    "                             scaleanchor='x',\n",
    "                             autorange=False),\n",
    "                  yaxis2=dict(tick0=0,\n",
    "                              range=(25, 0),\n",
    "                              scaleanchor='x2',\n",
    "                              autorange=False),\n",
    "                  hoverlabel=dict(font_size=18),\n",
    "                  height=700,\n",
    "                  margin=dict(r=0, l=0, b=0, t=0, pad=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save figure\n",
    "pio.write_html(fig,\n",
    "               file=f\"../_includes/figures/image_vs_pcd.html\",\n",
    "               full_html=False,\n",
    "               include_plotlyjs='cdn')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Point clouds as voxel grids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper functions\n",
    "def make_voxel(origin, index=0, voxel_size=1):\n",
    "    ox, oy, oz = origin\n",
    "    vs = voxel_size\n",
    "    \n",
    "    x = [ox, ox, ox+vs, ox+vs, ox, ox, ox+vs, ox+vs]\n",
    "    y = [oy, oy+vs, oy+vs, oy, oy, oy+vs, oy+vs, oy]\n",
    "    z = [oz, oz, oz, oz, oz+vs, oz+vs, oz+vs, oz+vs]\n",
    "    \n",
    "    i = (np.array([7, 0, 0, 0, 4, 4, 6, 6, 4, 0, 3, 2]) + index * 8).tolist()\n",
    "    j = (np.array([3, 4, 1, 2, 5, 6, 5, 2, 0, 1, 6, 3]) + index * 8).tolist()\n",
    "    k = (np.array([0, 7, 2, 3, 6, 7, 1, 1, 5, 5, 7, 6]) + index * 8).tolist()\n",
    "    \n",
    "    return x, y, z, i, j, k\n",
    "\n",
    "\n",
    "def get_voxels(voxel_grid):\n",
    "    x, y, z, i, j, k = [], [], [], [], [], []\n",
    "    for index, v in enumerate(voxel_grid.get_voxels()):\n",
    "        voxel = make_voxel(origin=v.grid_index, index=index)\n",
    "        x.extend(voxel[0])\n",
    "        y.extend(voxel[1])\n",
    "        z.extend(voxel[2])\n",
    "        i.extend(voxel[3])\n",
    "        j.extend(voxel[4])\n",
    "        k.extend(voxel[5])\n",
    "        \n",
    "    return x, y, z, i, j, k\n",
    "\n",
    "# Make voxel grid from point cloud and transform to mesh for visualization\n",
    "voxel_grid = o3d.geometry.VoxelGrid.create_from_point_cloud(pcd, voxel_size=0.003)\n",
    "x, y, z, i, j, k = get_voxels(voxel_grid)\n",
    "\n",
    "# Use Open3D to remove duplicate and faulty vertices and triangles\n",
    "vertices = np.vstack([x, y, z]).T\n",
    "triangles = np.vstack([i, j, k]).T\n",
    "\n",
    "mesh = o3d.geometry.TriangleMesh()\n",
    "mesh.vertices = o3d.utility.Vector3dVector(vertices)\n",
    "mesh.triangles = o3d.utility.Vector3iVector(triangles)\n",
    "\n",
    "mesh.remove_duplicated_vertices()\n",
    "mesh.remove_unreferenced_vertices()\n",
    "\n",
    "mesh.remove_duplicated_triangles()\n",
    "mesh.remove_degenerate_triangles()\n",
    "\n",
    "# Convert to Numpy for visualization\n",
    "vertices = np.asarray(mesh.vertices)\n",
    "triangles = np.asarray(mesh.triangles)\n",
    "x = vertices[:, 0]\n",
    "y = vertices[:, 1]\n",
    "z = vertices[:, 2]\n",
    "i = triangles[:, 0]\n",
    "j = triangles[:, 1]\n",
    "k = triangles[:, 2]\n",
    "\n",
    "fig = go.Figure(go.Mesh3d(x=x,\n",
    "                          y=y,\n",
    "                          z=z,\n",
    "                          i=i,\n",
    "                          j=j,\n",
    "                          k=k,\n",
    "                          hovertemplate=\"<b>Voxel</b><br>x: %{x}<br>y: %{y}<br>z: %{z}<extra></extra>\",\n",
    "                          hoverlabel=dict(bgcolor=\"white\"),\n",
    "                          colorscale=\"magma\",\n",
    "                          intensity=z,\n",
    "                          flatshading=True,\n",
    "                          showscale=False))\n",
    "\n",
    "# Viewpoint\n",
    "camera = dict(eye=dict(x=-2.1, y=0.7, z=1.2),\n",
    "              up=dict(x=0, y=1, z=0),\n",
    "              center=dict(x=0, y=0, z=0))\n",
    "\n",
    "fig.update_layout(scene=dict(\n",
    "                    xaxis=dict(visible=False),\n",
    "                    yaxis=dict(visible=False),\n",
    "                    zaxis=dict(visible=False),\n",
    "                    aspectmode='data'),\n",
    "                  height=700,\n",
    "                  margin=dict(r=0, l=0, b=0, t=0, pad=0),\n",
    "                  scene_camera=camera,\n",
    "                  scene_dragmode=\"orbit\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save figure\n",
    "pio.write_html(fig,\n",
    "               file=f\"../_includes/figures/pcd_as_voxel.html\",\n",
    "               full_html=False,\n",
    "               include_plotlyjs='cdn')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Point clouds as meshes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Best parameters (lowest density variance) were found using grid search between [1, 20]\n",
    "# for both parameters. Good normals are crucial for mesh reconstruction!\n",
    "print(\"Computing surface normals...\")\n",
    "pcd.estimate_normals(o3d.geometry.KDTreeSearchParamKNN(8))\n",
    "pcd.orient_normals_consistent_tangent_plane(8)\n",
    "\n",
    "# Increase `depth` to get finer details\n",
    "# This can take some time without a multi-core cpu\n",
    "print(\"Making triangle mesh...\")\n",
    "mesh, densities = o3d.geometry.TriangleMesh.create_from_point_cloud_poisson(pcd, depth=9, n_threads=16)\n",
    "mesh.remove_duplicated_vertices()\n",
    "mesh.remove_unreferenced_vertices()\n",
    "mesh.remove_duplicated_triangles()\n",
    "mesh.remove_degenerate_triangles()\n",
    "print(mesh)\n",
    "\n",
    "# Reduce resolution to reduce file size\n",
    "print(\"Decimating triangles...\")\n",
    "mesh = mesh.simplify_quadric_decimation(target_number_of_triangles=10000)\n",
    "print(mesh)\n",
    "\n",
    "vertices = np.asarray(mesh.vertices)\n",
    "triangles = np.asarray(mesh.triangles)\n",
    "x = vertices[:, 0]\n",
    "y = vertices[:, 1]\n",
    "z = vertices[:, 2]\n",
    "\n",
    "fig = go.Figure(ff.create_trisurf(x=x, y=y, z=z,\n",
    "                                  simplices=triangles,\n",
    "                                  plot_edges=True,\n",
    "                                  edges_color=\"black\",\n",
    "                                  colormap=['#000004', '#180f3d', '#440f76', '#721f81', '#9e2f7f',\n",
    "                                            '#cd4071', '#f1605d', '#fd9668', '#feca8d', '#fcfdbf'],\n",
    "                                  show_colorbar=False).data)\n",
    "\n",
    "# The figure factory creates two plots: 1. The triangle model, 2. The mesh.\n",
    "# Visibility is set here changed in the visualization using the buttons.\n",
    "buttons = [dict(label=\"w/ mesh\", method=\"update\", args=[dict(visible=[True, True])]),\n",
    "           dict(label=\"w/o mesh\", method=\"update\", args=[dict(visible=[True, False])]),\n",
    "           dict(label=\"wireframe\", method=\"update\", args=[dict(visible=[False, True])])]\n",
    "\n",
    "# W/o figure factory. No wireframe though.\n",
    "\"\"\"\n",
    "i = triangles[:, 0]\n",
    "j = triangles[:, 1]\n",
    "k = triangles[:, 2]\n",
    "\n",
    "fig = go.Figure(go.Mesh3d(x=x, y=y, z=z,\n",
    "                          i=i, j=j, k=k,\n",
    "                          colorscale=\"magma\",\n",
    "                          intensity=z,\n",
    "                          showscale=False))\n",
    "\"\"\"\n",
    "\n",
    "# Viewpoint\n",
    "camera = dict(eye=dict(x=-2.1, y=0.7, z=1.2),\n",
    "              up=dict(x=0, y=1, z=0),\n",
    "              center=dict(x=0, y=0, z=0))\n",
    "\n",
    "fig.update_layout(scene=dict(\n",
    "                    xaxis=dict(visible=False),\n",
    "                    yaxis=dict(visible=False),\n",
    "                    zaxis=dict(visible=False),\n",
    "                    aspectmode='data'),\n",
    "                  height=700,\n",
    "                  margin=dict(r=0, l=0, b=0, t=0, pad=0),\n",
    "                  scene_camera=camera,\n",
    "                  scene_dragmode=\"orbit\",\n",
    "                  updatemenus=[dict(buttons=buttons, x=0.2, y=1)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save figure\n",
    "pio.write_html(fig,\n",
    "               file=f\"../_includes/figures/pcd_as_mesh.html\",\n",
    "               full_html=False,\n",
    "               include_plotlyjs='cdn')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}