--[Part 2 of 2] Semantic region building

examples/viewer.py

@@ -76,6 +76,9 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None:
         self.debug_bullet_draw = False
         # draw active contact point debug line visualizations
         self.contact_debug_draw = False
+        # draw semantic region debug visualizations if present
+        self.semantic_region_debug_draw = False
+
         # cache most recently loaded URDF file for quick-reload
         self.cached_urdf = ""
 
@@ -176,6 +179,7 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None:
         self.replay_renderer_cfg: Optional[ReplayRendererConfiguration] = None
         self.replay_renderer: Optional[ReplayRenderer] = None
         self.reconfigure_sim()
+        self.debug_semantic_colors = {}
 
         # compute NavMesh if not already loaded by the scene.
         if (
@@ -189,40 +193,54 @@ def __init__(self, sim_settings: Dict[str, Any]) -> None:
         logger.setLevel("INFO")
         self.print_help_text()
 
-    def draw_contact_debug(self):
+    def draw_contact_debug(self, debug_line_render: Any):
         """
         This method is called to render a debug line overlay displaying active contact points and normals.
         Yellow lines show the contact distance along the normal and red lines show the contact normal at a fixed length.
         """
         yellow = mn.Color4.yellow()
         red = mn.Color4.red()
         cps = self.sim.get_physics_contact_points()
-        self.sim.get_debug_line_render().set_line_width(1.5)
+        debug_line_render.set_line_width(1.5)
         camera_position = self.render_camera.render_camera.node.absolute_translation
         # only showing active contacts
         active_contacts = (x for x in cps if x.is_active)
         for cp in active_contacts:
             # red shows the contact distance
-            self.sim.get_debug_line_render().draw_transformed_line(
+            debug_line_render.draw_transformed_line(
                 cp.position_on_b_in_ws,
                 cp.position_on_b_in_ws
                 + cp.contact_normal_on_b_in_ws * -cp.contact_distance,
                 red,
             )
             # yellow shows the contact normal at a fixed length for visualization
-            self.sim.get_debug_line_render().draw_transformed_line(
+            debug_line_render.draw_transformed_line(
                 cp.position_on_b_in_ws,
                 # + cp.contact_normal_on_b_in_ws * cp.contact_distance,
                 cp.position_on_b_in_ws + cp.contact_normal_on_b_in_ws * 0.1,
                 yellow,
             )
-            self.sim.get_debug_line_render().draw_circle(
+            debug_line_render.draw_circle(
                 translation=cp.position_on_b_in_ws,
                 radius=0.005,
                 color=yellow,
                 normal=camera_position - cp.position_on_b_in_ws,
             )
 
+    def draw_region_debug(self, debug_line_render: Any) -> None:
+        """
+        Draw the semantic region wireframes.
+        """
+
+        for region in self.sim.semantic_scene.regions:
+            color = self.debug_semantic_colors.get(region.id, mn.Color4.magenta())
+            for edge in region.volume_edges:
+                debug_line_render.draw_transformed_line(
+                    edge[0],
+                    edge[1],
+                    color,
+                )
+
     def debug_draw(self):
         """
         Additional draw commands to be called during draw_event.
@@ -231,8 +249,18 @@ def debug_draw(self):
             render_cam = self.render_camera.render_camera
             proj_mat = render_cam.projection_matrix.__matmul__(render_cam.camera_matrix)
             self.sim.physics_debug_draw(proj_mat)
+
+        debug_line_render = self.sim.get_debug_line_render()
         if self.contact_debug_draw:
-            self.draw_contact_debug()
+            self.draw_contact_debug(debug_line_render)
+
+        if self.semantic_region_debug_draw:
+            if len(self.debug_semantic_colors) != len(self.sim.semantic_scene.regions):
+                for region in self.sim.semantic_scene.regions:
+                    self.debug_semantic_colors[region.id] = mn.Color4(
+                        mn.Vector3(np.random.random(3))
+                    )
+            self.draw_region_debug(debug_line_render)
 
     def draw_event(
         self,
@@ -473,6 +501,12 @@ def key_press_event(self, event: Application.KeyEvent) -> None:
 
         elif key == pressed.H:
             self.print_help_text()
+        elif key == pressed.J:
+            logger.info(
+                f"Toggle Region Draw from {self.semantic_region_debug_draw } to {not self.semantic_region_debug_draw}"
+            )
+            # Toggle visualize semantic bboxes. Currently only regions supported
+            self.semantic_region_debug_draw = not self.semantic_region_debug_draw
 
         elif key == pressed.TAB:
             # NOTE: (+ALT) - reconfigure without cycling scenes
@@ -590,7 +624,6 @@ def key_press_event(self, event: Application.KeyEvent) -> None:
         elif key == pressed.V:
             self.invert_gravity()
             logger.info("Command: gravity inverted")
-
         elif key == pressed.N:
             # (default) - toggle navmesh visualization
             # NOTE: (+ALT) - re-sample the agent position on the NavMesh
@@ -973,6 +1006,7 @@ def print_help_text(self) -> None:
     ',':        Render a Bullet collision shape debug wireframe overlay (white=active, green=sleeping, blue=wants sleeping, red=can't sleep).
     'c':        Run a discrete collision detection pass and render a debug wireframe overlay showing active contact points and normals (yellow=fixed length normals, red=collision distances).
                 (+SHIFT) Toggle the contact point debug render overlay on/off.
+    'j'         Toggle Semantic visualization bounds (currently only Semantic Region annotations)
 
     Object Interactions:
     SPACE:      Toggle physics simulation on/off.

src/esp/assets/ResourceManager.cpp

@@ -305,77 +305,30 @@ std::vector<std::string> ResourceManager::buildVertexColorMapReport(
       semanticScene_);
 }  // ResourceManager::buildVertexColorMapReport
 
-bool ResourceManager::loadSemanticSceneDescriptor(
-    const std::string& ssdFilename,
+bool ResourceManager::loadSemanticScene(
+    const std::shared_ptr<metadata::attributes::SemanticAttributes>&
+        semanticAttr,
     const std::string& activeSceneName) {
-  namespace FileUtil = Cr::Utility::Path;
+  const std::string ssdFilename =
+      semanticAttr != nullptr ? semanticAttr->getSemanticDescriptorFilename()
+                              : "";
   semanticScene_ = nullptr;
-  if (ssdFilename != "") {
+  if ((ssdFilename != "") || ((semanticAttr != nullptr) &&
+                              (semanticAttr->getNumRegionInstances() > 0))) {
     bool success = false;
-    // semantic scene descriptor might not exist
+    // semantic scene descriptor might not exist so (re)create it.
     semanticScene_ = scene::SemanticScene::create();
-    ESP_DEBUG(Mn::Debug::Flag::NoSpace)
-        << "SceneInstance : `" << activeSceneName
-        << "` proposed Semantic Scene Descriptor filename : `" << ssdFilename
-        << "`.";
-
-    bool fileExists = FileUtil::exists(ssdFilename);
-    if (fileExists) {
-      // Attempt to load semantic scene descriptor specified in scene instance
-      // file, agnostic to file type inferred by name, if file exists.
-      success = scene::SemanticScene::loadSemanticSceneDescriptor(
-          ssdFilename, *semanticScene_);
-      if (success) {
-        ESP_DEBUG(Mn::Debug::Flag::NoSpace)
-            << "SSD with SceneInstanceAttributes-provided name `" << ssdFilename
-            << "` successfully found and loaded.";
-      } else {
-        // here if provided file exists but does not correspond to appropriate
-        // SSD
-        ESP_ERROR(Mn::Debug::Flag::NoSpace)
-            << "SSD Load Failure! File with "
-               "SceneInstanceAttributes-provided name `"
-            << ssdFilename << "` exists but failed to load.";
-      }
-      return success;
-      // if not success then try to construct a name
-    } else {
-      // attempt to look for specified file failed, attempt to build new file
-      // name by searching in path specified of specified file for
-      // info_semantic.json file for replica dataset
-      const std::string constructedFilename = FileUtil::join(
-          FileUtil::split(ssdFilename).first(), "info_semantic.json");
-      fileExists = FileUtil::exists(constructedFilename);
-      if (fileExists) {
-        success = scene::SemanticScene::loadReplicaHouse(constructedFilename,
-                                                         *semanticScene_);
-        if (success) {
-          ESP_DEBUG(Mn::Debug::Flag::NoSpace)
-              << "SSD for Replica using constructed file : `"
-              << constructedFilename << "` in directory with `" << ssdFilename
-              << "` loaded successfully";
-        } else {
-          // here if constructed file exists but does not correspond to
-          // appropriate SSD or some loading error occurred.
-          ESP_ERROR(Mn::Debug::Flag::NoSpace)
-              << "SSD Load Failure! Replica file with constructed name `"
-              << ssdFilename << "` exists but failed to load.";
-        }
-        return success;
-      } else {
-        // neither provided non-empty filename nor constructed filename
-        // exists. This is probably due to an incorrect naming in the
-        // SceneInstanceAttributes
-        ESP_WARNING(Mn::Debug::Flag::NoSpace)
-            << "SSD File Naming Issue! Neither "
-               "SceneInstanceAttributes-provided name : `"
-            << ssdFilename << "` nor constructed filename : `"
-            << constructedFilename << "` exist on disk.";
-        return false;
-      }
-    }  // if given SSD file name specified exists
-  }    // if semantic scene descriptor specified in scene instance
 
+    success = scene::SemanticScene::loadSemanticSceneDescriptor(
+        semanticAttr, *semanticScene_);
+
+    ESP_VERY_VERBOSE(Mn::Debug::Flag::NoSpace)
+        << "Attempt to create SemanticScene for Current Scene :`"
+        << activeSceneName << "` with ssdFilename : `" << ssdFilename
+        << "` :" << (success ? " " : "Not ") << "Successful";
+
+    return success;
+  }
   return false;
 }  // ResourceManager::loadSemanticSceneDescriptor
 

src/esp/assets/ResourceManager.h

@@ -52,6 +52,7 @@ namespace attributes {
 class ObjectAttributes;
 class PbrShaderAttributes;
 class PhysicsManagerAttributes;
+class SemanticAttributes;
 class SceneObjectInstanceAttributes;
 class StageAttributes;
 }  // namespace attributes
@@ -193,15 +194,19 @@ class ResourceManager {
   bool semanticSceneExists() const { return (semanticScene_ != nullptr); }
 
   /**
-   * @brief Load semantic scene descriptor file specified by @p ssdFilename ,
-   * for the passed @p activeSceneName .
-   * @param ssdFilename The fully qualified filename candidate for the ssd file.
+   * @brief Load semantic scene data from descriptor file and metadata specified
+   * in @p semanticAttr , for the passed @p activeSceneName .
+   * @param semanticAttr Pointer to semantic attributes, if they exist. This
+   * will hold fully-qualified filename along with other attributes required to
+   * create the semantic scene.
    * @param activeSceneName Name of the currently active scene that we will be
    * loading the SSD for.
    * @return whether loaded successfully or not.
    */
-  bool loadSemanticSceneDescriptor(const std::string& ssdFilename,
-                                   const std::string& activeSceneName);
+  bool loadSemanticScene(
+      const std::shared_ptr<metadata::attributes::SemanticAttributes>&
+          semanticAttr,
+      const std::string& activeSceneName);
 
   /**
    * @brief Load a scene mesh and add it to the specified @ref DrawableGroup as

src/esp/bindings/SceneBindings.cpp

@@ -159,13 +159,30 @@ void initSceneBindings(py::module& m) {
   semanticRegion
       .def_property_readonly(
           "id", &SemanticRegion::id,
-          "The ID of the region, of the form ``<level_id>_<region_id>``")
+          "The ID of the region, either as the region's unique name, or of the "
+          "form ``<level_id>_<region_id>``")
       .def_property_readonly("level", &SemanticRegion::level)
       .def_property_readonly("aabb", &SemanticRegion::aabb)
       .def_property_readonly("category", &SemanticRegion::category,
                              "The semantic category of the region")
       .def_property_readonly("objects", &SemanticRegion::objects,
-                             "All objects in the region");
+                             "All objects in the region")
+      .def_property_readonly("poly_loop_points",
+                             &SemanticRegion::getPolyLoopPoints,
+                             "The points making up the polyloop for this "
+                             "region, coplanar and parallel to the floor.")
+      .def_property_readonly(
+          "volume_edges", &SemanticRegion::getVisEdges,
+          "The edges, as pairs of points, that determine "
+          "the boundaries of the region. For visualizations.")
+      .def_property_readonly("floor_height", &SemanticRegion::getFloorHeight,
+                             "The height above the x-z plane for the floor of "
+                             "the semantic region.")
+      .def_property_readonly("extrusion_height",
+                             &SemanticRegion::getExtrusionHeight,
+                             "The height of the extrusion above the floor.")
+      .def("contains", &SemanticRegion::contains, "point"_a,
+           "Check whether the given point is contained in the given region.");
 
   // ==== SemanticObject ====
   semanticObject

src/esp/metadata/attributes/SemanticAttributes.cpp

@@ -8,17 +8,17 @@ namespace metadata {
 namespace attributes {
 
 /////////////////////////////////
-// SemanticRegionAttributes
+// SemanticVolumeAttributes
 
-SemanticRegionAttributes::SemanticRegionAttributes(const std::string& handle)
-    : AbstractAttributes("SemanticRegionAttributes", handle) {
+SemanticVolumeAttributes::SemanticVolumeAttributes(const std::string& handle)
+    : AbstractAttributes("SemanticVolumeAttributes", handle) {
   // Initialize values
   set("floor_height", 0.0);
   set("extrusion_height", 2.5);
 
-}  // SemanticRegionAttributes ctor
+}  // SemanticVolumeAttributes ctor
 
-std::string SemanticRegionAttributes::getObjectInfoHeaderInternal() const {
+std::string SemanticVolumeAttributes::getObjectInfoHeaderInternal() const {
   std::string res =
       "Name,Label,Floor Height,Extrusion Height,Min Bounds,Max Bounds,";
   int iter = 0;
@@ -29,7 +29,7 @@ std::string SemanticRegionAttributes::getObjectInfoHeaderInternal() const {
   return res;
 }
 
-std::string SemanticRegionAttributes::getObjectInfoInternal() const {
+std::string SemanticVolumeAttributes::getObjectInfoInternal() const {
   std::string res = Cr::Utility::formatString(
       "{},{},{},{},{},{},", getAsString("handle"), getAsString("label"),
       getAsString("floor_height"), getAsString("extrusion_height"),
@@ -42,12 +42,12 @@ std::string SemanticRegionAttributes::getObjectInfoInternal() const {
   Cr::Utility::formatInto(res, res.size(), "],");
 
   return res;
-}  // SemanticRegionAttributes::getObjectInfoInternal()
+}  // SemanticVolumeAttributes::getObjectInfoInternal()
 
-void SemanticRegionAttributes::writeValuesToJson(
+void SemanticVolumeAttributes::writeValuesToJson(
     io::JsonGenericValue& jsonObj,
     io::JsonAllocator& allocator) const {
-  // map "handle" to "name" key in json - this SemanticRegionAttributes' handle
+  // map "handle" to "name" key in json - this SemanticVolumeAttributes' handle
   // is its unique name
   writeValueToJson("handle", "name", jsonObj, allocator);
   writeValueToJson("label", jsonObj, allocator);
@@ -60,7 +60,7 @@ void SemanticRegionAttributes::writeValuesToJson(
     io::addMember(jsonObj, "poly_loop", polyLoop_, allocator);
   }
 
-}  // SemanticRegionAttributes::writeValuesToJson
+}  // SemanticVolumeAttributes::writeValuesToJson
 
 /////////////////////////////////
 // SemanticAttributes
@@ -85,7 +85,7 @@ SemanticAttributes::SemanticAttributes(const SemanticAttributes& otr)
       availableRegionInstIDs_(otr.availableRegionInstIDs_) {
   // get refs to internal subconfigs for semantic region attributes
   regionAnnotationConfig_ = editSubconfig<Configuration>("region_annotations");
-  copySubconfigIntoMe<SemanticRegionAttributes>(otr.regionAnnotationConfig_,
+  copySubconfigIntoMe<SemanticVolumeAttributes>(otr.regionAnnotationConfig_,
                                                 regionAnnotationConfig_);
 }  // SemanticAttributes copy ctor
 
@@ -162,7 +162,7 @@ SemanticAttributes& SemanticAttributes::operator=(
     // get refs to internal subconfigs for semantic region attributes
     regionAnnotationConfig_ =
         editSubconfig<Configuration>("region_annotations");
-    copySubconfigIntoMe<SemanticRegionAttributes>(otr.regionAnnotationConfig_,
+    copySubconfigIntoMe<SemanticVolumeAttributes>(otr.regionAnnotationConfig_,
                                                   regionAnnotationConfig_);
   }
   return *this;