Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add DebugLineRender utility #1349

Merged
merged 10 commits into from
Jul 1, 2021
Merged

Add DebugLineRender utility #1349

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
473bb7f
add DebugLineRender util for thick-line-rendering
eundersander Jun 17, 2021
ce39983
integrate DebugLineRender into Simulator
eundersander Jun 28, 2021
3cb7729
Python bindings for DebugLineRender
eundersander Jun 28, 2021
7e1f726
update replay_tutorial.py: debug line rendering; brighter lights
eundersander Jun 28, 2021
2204df6
fix for logging include
eundersander Jun 28, 2021
8dbd74e
polish
eundersander Jun 28, 2021
8efc781
compile fix for JS build; fix deprecation warning
eundersander Jun 28, 2021
fc3540e
switch to FlatGL; switch to ArrayView for points list; other polish
eundersander Jun 29, 2021
6ba198a
version of drawPathWithEndpointCircles that takes a std::vector, for …
eundersander Jun 29, 2021
2dfbdbb
update comment
eundersander Jun 29, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
102 changes: 78 additions & 24 deletions examples/tutorials/colabs/replay_tutorial.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@
"import numpy as np\n",
"\n",
"import habitat_sim\n",
"from habitat_sim.gfx import LightInfo, LightPositionModel\n",
"from habitat_sim.utils import gfx_replay_utils\n",
"from habitat_sim.utils import viz_utils as vut\n",
"\n",
Expand Down Expand Up @@ -116,7 +117,9 @@
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"metadata": {
"lines_to_next_cell": 2
},
"outputs": [],
"source": [
"\n",
Expand Down Expand Up @@ -180,6 +183,28 @@
"metadata": {},
"outputs": [],
"source": [
"\n",
"\n",
"def configure_lighting(sim):\n",
" light_setup = [\n",
" LightInfo(\n",
" vector=[1.0, 1.0, 0.0, 1.0],\n",
" color=[18.0, 18.0, 18.0],\n",
" model=LightPositionModel.Global,\n",
" ),\n",
" LightInfo(\n",
" vector=[0.0, -1.0, 0.0, 1.0],\n",
" color=[5.0, 5.0, 5.0],\n",
" model=LightPositionModel.Global,\n",
" ),\n",
" LightInfo(\n",
" vector=[-1.0, 1.0, 1.0, 1.0],\n",
" color=[18.0, 18.0, 18.0],\n",
" model=LightPositionModel.Global,\n",
" ),\n",
" ]\n",
" sim.set_light_setup(light_setup)\n",
"\n",
"\n",
"if __name__ == \"__main__\":\n",
" import argparse\n",
Expand Down Expand Up @@ -207,6 +232,8 @@
"else:\n",
" sim.reconfigure(cfg)\n",
"\n",
"configure_lighting(sim)\n",
"\n",
"agent_state = habitat_sim.AgentState()\n",
"agent = sim.initialize_agent(0, agent_state)"
]
Expand Down Expand Up @@ -403,6 +430,8 @@
"metadata": {},
"outputs": [],
"source": [
"\n",
"sim.close()\n",
"\n",
"# use same agents/sensors from earlier, with different backend config\n",
"playback_cfg = habitat_sim.Configuration(\n",
Expand All @@ -412,13 +441,13 @@
" cfg.agents,\n",
")\n",
"\n",
"sim.close()\n",
"\n",
"if not sim:\n",
" sim = habitat_sim.Simulator(playback_cfg)\n",
"else:\n",
" sim.reconfigure(playback_cfg)\n",
"\n",
"configure_lighting(sim)\n",
"\n",
"agent_state = habitat_sim.AgentState()\n",
"sim.initialize_agent(0, agent_state)\n",
"\n",
Expand Down Expand Up @@ -563,32 +592,59 @@
"sensor_node.translation = [-1.1, -0.9, -0.2]\n",
"sensor_node.rotation = mn.Quaternion.rotation(mn.Deg(-115), mn.Vector3(0.0, 1.0, 0))\n",
"\n",
"prim_attr_mgr = sim.get_asset_template_manager()\n",
"# get the rigid object manager, which provides direct\n",
"# access to objects\n",
"rigid_obj_mgr = sim.get_rigid_object_manager()\n",
"# visualize the recorded agent transform as a cylinder\n",
"agent_viz_handle = prim_attr_mgr.get_template_handles(\"cylinderSolid\")[0]\n",
"agent_viz_obj = rigid_obj_mgr.add_object_by_template_handle(agent_viz_handle)\n",
"agent_viz_obj.motion_type = habitat_sim.physics.MotionType.KINEMATIC\n",
"agent_viz_obj.collidable = False\n",
"\n",
"# visualize the recorded sensor transform as a cube\n",
"sensor_viz_handle = prim_attr_mgr.get_template_handles(\"cubeSolid\")[0]\n",
"sensor_viz_obj = rigid_obj_mgr.add_object_by_template_handle(sensor_viz_handle)\n",
"sensor_viz_obj.motion_type = habitat_sim.physics.MotionType.KINEMATIC\n",
"sensor_viz_obj.collidable = False\n",
"# gather the agent trajectory for later visualization\n",
"agent_trajectory_points = []\n",
"for frame in range(player.get_num_keyframes()):\n",
" player.set_keyframe_index(frame)\n",
" (agent_translation, _) = player.get_user_transform(\"agent\")\n",
" agent_trajectory_points.append(agent_translation)\n",
"\n",
"debug_line_render = sim.get_debug_line_render()\n",
"debug_line_render.set_line_width(2.0)\n",
"agent_viz_box = mn.Range3D(mn.Vector3(-0.1, 0.0, -0.1), mn.Vector3(0.1, 0.4, 0.1))\n",
"sensor_viz_box = mn.Range3D(mn.Vector3(-0.1, -0.1, -0.1), mn.Vector3(0.1, 0.1, 0.1))\n",
"\n",
"for frame in range(player.get_num_keyframes()):\n",
" player.set_keyframe_index(frame)\n",
"\n",
" (agent_translation, agent_rotation) = player.get_user_transform(\"agent\")\n",
" agent_viz_obj.translation = agent_translation\n",
" agent_viz_obj.rotation = agent_rotation\n",
"\n",
" rot_mat = agent_rotation.to_matrix()\n",
" full_mat = mn.Matrix4.from_(rot_mat, agent_translation)\n",
"\n",
" # draw a box in the agent body's local space\n",
" debug_line_render.push_transform(full_mat)\n",
" debug_line_render.draw_box(\n",
" agent_viz_box.min, agent_viz_box.max, mn.Color4(1.0, 0.0, 0.0, 1.0)\n",
" )\n",
" debug_line_render.pop_transform()\n",
"\n",
" for (radius, opacity) in [(0.2, 0.6), (0.25, 0.4), (0.3, 0.2)]:\n",
" debug_line_render.draw_circle(\n",
" agent_translation, radius, mn.Color4(0.0, 1.0, 1.0, opacity)\n",
" )\n",
"\n",
" # draw a box in the sensor's local space\n",
" (sensor_translation, sensor_rotation) = player.get_user_transform(\"sensor\")\n",
" sensor_viz_obj.translation = sensor_translation\n",
" sensor_viz_obj.rotation = sensor_rotation\n",
" debug_line_render.push_transform(\n",
" mn.Matrix4.from_(sensor_rotation.to_matrix(), sensor_translation)\n",
" )\n",
" debug_line_render.draw_box(\n",
" sensor_viz_box.min, sensor_viz_box.max, mn.Color4(1.0, 0.0, 0.0, 1.0)\n",
" )\n",
" # draw a line in the sensor look direction (-z in local space)\n",
" debug_line_render.draw_transformed_line(\n",
" mn.Vector3.zero_init(),\n",
" mn.Vector3(0.0, 0.0, -0.5),\n",
" mn.Color4(1.0, 0.0, 0.0, 1.0),\n",
" mn.Color4(1.0, 1.0, 1.0, 1.0),\n",
" )\n",
" debug_line_render.pop_transform()\n",
"\n",
" # draw the agent trajectory\n",
" debug_line_render.draw_path_with_endpoint_circles(\n",
" agent_trajectory_points, 0.07, mn.Color4(1.0, 1.0, 1.0, 1.0)\n",
" )\n",
"\n",
" observations.append(sim.get_sensor_observations())\n",
"\n",
Expand All @@ -601,8 +657,6 @@
" open_vid=show_video,\n",
" )\n",
"\n",
"rigid_obj_mgr.remove_object_by_id(agent_viz_obj.object_id)\n",
"rigid_obj_mgr.remove_object_by_id(sensor_viz_obj.object_id)\n",
"\n",
"# clean up the player\n",
"player.close()"
Expand Down
98 changes: 75 additions & 23 deletions examples/tutorials/nb_python/replay_tutorial.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@
import numpy as np

import habitat_sim
from habitat_sim.gfx import LightInfo, LightPositionModel
from habitat_sim.utils import gfx_replay_utils
from habitat_sim.utils import viz_utils as vut

Expand Down Expand Up @@ -144,6 +145,28 @@ def simulate_with_moving_agent(
# ## More tutorial setup
# %%


def configure_lighting(sim):
light_setup = [
LightInfo(
vector=[1.0, 1.0, 0.0, 1.0],
color=[18.0, 18.0, 18.0],
model=LightPositionModel.Global,
),
LightInfo(
vector=[0.0, -1.0, 0.0, 1.0],
color=[5.0, 5.0, 5.0],
model=LightPositionModel.Global,
),
LightInfo(
vector=[-1.0, 1.0, 1.0, 1.0],
color=[18.0, 18.0, 18.0],
model=LightPositionModel.Global,
),
]
sim.set_light_setup(light_setup)


if __name__ == "__main__":
import argparse

Expand All @@ -170,6 +193,8 @@ def simulate_with_moving_agent(
else:
sim.reconfigure(cfg)

configure_lighting(sim)

agent_state = habitat_sim.AgentState()
agent = sim.initialize_agent(0, agent_state)

Expand Down Expand Up @@ -276,6 +301,8 @@ def simulate_with_moving_agent(
# Note call to gfx_replay_utils.make_backend_configuration_for_playback. Note that we don't specify a scene or stage when reconfiguring for replay playback. need_separate_semantic_scene_graph is generally set to False. If you're using a semantic sensor and replaying a scene that uses a separate semantic mesh (like an MP3D scene), set this to True. If in doubt, be aware there's a Habitat runtime warning that will always catch incorrect usage of this flag.
# %%

sim.close()

# use same agents/sensors from earlier, with different backend config
playback_cfg = habitat_sim.Configuration(
gfx_replay_utils.make_backend_configuration_for_playback(
Expand All @@ -284,13 +311,13 @@ def simulate_with_moving_agent(
cfg.agents,
)

sim.close()

if not sim:
sim = habitat_sim.Simulator(playback_cfg)
else:
sim.reconfigure(playback_cfg)

configure_lighting(sim)

agent_state = habitat_sim.AgentState()
sim.initialize_agent(0, agent_state)

Expand Down Expand Up @@ -370,32 +397,59 @@ def simulate_with_moving_agent(
sensor_node.translation = [-1.1, -0.9, -0.2]
sensor_node.rotation = mn.Quaternion.rotation(mn.Deg(-115), mn.Vector3(0.0, 1.0, 0))

prim_attr_mgr = sim.get_asset_template_manager()
# get the rigid object manager, which provides direct
# access to objects
rigid_obj_mgr = sim.get_rigid_object_manager()
# visualize the recorded agent transform as a cylinder
agent_viz_handle = prim_attr_mgr.get_template_handles("cylinderSolid")[0]
agent_viz_obj = rigid_obj_mgr.add_object_by_template_handle(agent_viz_handle)
agent_viz_obj.motion_type = habitat_sim.physics.MotionType.KINEMATIC
agent_viz_obj.collidable = False

# visualize the recorded sensor transform as a cube
sensor_viz_handle = prim_attr_mgr.get_template_handles("cubeSolid")[0]
sensor_viz_obj = rigid_obj_mgr.add_object_by_template_handle(sensor_viz_handle)
sensor_viz_obj.motion_type = habitat_sim.physics.MotionType.KINEMATIC
sensor_viz_obj.collidable = False
# gather the agent trajectory for later visualization
agent_trajectory_points = []
for frame in range(player.get_num_keyframes()):
player.set_keyframe_index(frame)
(agent_translation, _) = player.get_user_transform("agent")
agent_trajectory_points.append(agent_translation)

debug_line_render = sim.get_debug_line_render()
debug_line_render.set_line_width(2.0)
agent_viz_box = mn.Range3D(mn.Vector3(-0.1, 0.0, -0.1), mn.Vector3(0.1, 0.4, 0.1))
sensor_viz_box = mn.Range3D(mn.Vector3(-0.1, -0.1, -0.1), mn.Vector3(0.1, 0.1, 0.1))

for frame in range(player.get_num_keyframes()):
player.set_keyframe_index(frame)

(agent_translation, agent_rotation) = player.get_user_transform("agent")
agent_viz_obj.translation = agent_translation
agent_viz_obj.rotation = agent_rotation

rot_mat = agent_rotation.to_matrix()
full_mat = mn.Matrix4.from_(rot_mat, agent_translation)

# draw a box in the agent body's local space
debug_line_render.push_transform(full_mat)
debug_line_render.draw_box(
agent_viz_box.min, agent_viz_box.max, mn.Color4(1.0, 0.0, 0.0, 1.0)
)
debug_line_render.pop_transform()

for (radius, opacity) in [(0.2, 0.6), (0.25, 0.4), (0.3, 0.2)]:
debug_line_render.draw_circle(
agent_translation, radius, mn.Color4(0.0, 1.0, 1.0, opacity)
)

# draw a box in the sensor's local space
(sensor_translation, sensor_rotation) = player.get_user_transform("sensor")
sensor_viz_obj.translation = sensor_translation
sensor_viz_obj.rotation = sensor_rotation
debug_line_render.push_transform(
mn.Matrix4.from_(sensor_rotation.to_matrix(), sensor_translation)
)
debug_line_render.draw_box(
sensor_viz_box.min, sensor_viz_box.max, mn.Color4(1.0, 0.0, 0.0, 1.0)
)
# draw a line in the sensor look direction (-z in local space)
debug_line_render.draw_transformed_line(
mn.Vector3.zero_init(),
mn.Vector3(0.0, 0.0, -0.5),
mn.Color4(1.0, 0.0, 0.0, 1.0),
mn.Color4(1.0, 1.0, 1.0, 1.0),
)
debug_line_render.pop_transform()

# draw the agent trajectory
debug_line_render.draw_path_with_endpoint_circles(
agent_trajectory_points, 0.07, mn.Color4(1.0, 1.0, 1.0, 1.0)
)

observations.append(sim.get_sensor_observations())

Expand All @@ -408,8 +462,6 @@ def simulate_with_moving_agent(
open_vid=show_video,
)

rigid_obj_mgr.remove_object_by_id(agent_viz_obj.object_id)
rigid_obj_mgr.remove_object_by_id(sensor_viz_obj.object_id)

# clean up the player
player.close()
Expand Down
38 changes: 38 additions & 0 deletions src/esp/bindings/GfxBindings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "python/corrade/EnumOperators.h"

#include "esp/assets/ResourceManager.h"
#include "esp/gfx/DebugLineRender.h"
#include "esp/gfx/LightSetup.h"
#include "esp/gfx/RenderCamera.h"
#include "esp/gfx/RenderTarget.h"
Expand Down Expand Up @@ -205,6 +206,43 @@ void initGfxBindings(py::module& m) {
.def(py::self == py::self)
.def(py::self != py::self);

py::class_<DebugLineRender, std::shared_ptr<DebugLineRender>>(
m, "DebugLineRender")
.def("set_line_width", &DebugLineRender::setLineWidth,
R"(See push_transform.)")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Update this doc? I assume it applied to lines following the command?

.def(
"push_transform", &DebugLineRender::pushTransform,
R"(Push (multiply) a transform onto the transform stack, affecting all line-drawing until popped. Must be paired with popTransform().)")
.def("pop_transform", &DebugLineRender::popTransform,
R"(See push_transform.)")
.def("draw_box", &DebugLineRender::drawBox,
R"(Draw a box in world-space or local-space (see pushTransform).)")
Comment on lines +219 to +220
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No color for box bindings?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

draw_box takes two Vector3 and a color. This binding here just takes all the C++ function parameters as-is without explicitly declaring them all here.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess this just makes it harder for python only user seeing the doc-string API reference to know the parameters. Also, does implicit conversion allow keyword (named) arguments?

.def(
"draw_circle", &DebugLineRender::drawCircle, "translation"_a,
"radius"_a, "color"_a, "num_segments"_a = 24,
"normal"_a = Magnum::Vector3{0.0, 1.0, 0.0},
R"(Draw a circle in world-space or local-space (see pushTransform). The circle is an approximation; see numSegments.)")
.def(
"draw_transformed_line",
py::overload_cast<const Magnum::Vector3&, const Magnum::Vector3&,
const Magnum::Color4&, const Magnum::Color4&>(
&DebugLineRender::drawTransformedLine),
"from"_a, "to"_a, "from_color"_a, "to_color"_a,
R"(Draw a line segment in world-space or local-space (see pushTransform).)")
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
R"(Draw a line segment in world-space or local-space (see pushTransform).)")
R"(Draw a line segment in world-space or local-space (see pushTransform) with interpolated color.)")

.def(
"draw_transformed_line",
py::overload_cast<const Magnum::Vector3&, const Magnum::Vector3&,
const Magnum::Color4&>(
&DebugLineRender::drawTransformedLine),
"from"_a, "to"_a, "color"_a,
R"(Draw a line segment in world-space or local-space (see pushTransform).)")
.def(
"draw_path_with_endpoint_circles",
&DebugLineRender::drawPathWithEndpointCircles, "points"_a, "radius"_a,
"color"_a, "num_segments"_a = 24,
"normal"_a = Magnum::Vector3{0.0, 1.0, 0.0},
R"(Draw a sequence of line segments with circles at the two endpoints. In world-space or local-space (see pushTransform).)");

m.attr("DEFAULT_LIGHTING_KEY") = DEFAULT_LIGHTING_KEY;
m.attr("NO_LIGHT_KEY") = NO_LIGHT_KEY;
}
Expand Down
5 changes: 4 additions & 1 deletion src/esp/bindings/SimBindings.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,10 @@ void initSimBindings(py::module& m) {
&Simulator::getRigidConstraintSettings, "constraint_id"_a,
R"(Get a copy of the settings for an existing rigid constraint.)")
.def("remove_rigid_constraint", &Simulator::removeRigidConstraint,
"constraint_id"_a, R"(Remove a rigid constraint by id.)");
"constraint_id"_a, R"(Remove a rigid constraint by id.)")
.def("get_debug_line_render", &Simulator::getDebugLineRender,
pybind11::return_value_policy::reference,
R"(Get visualization helper for rendering lines.)");
}

} // namespace sim
Expand Down