Automatically supply scenario path to Envision server

CHANGELOG.md

@@ -11,6 +11,7 @@ Copy and pasting the git commit messages is __NOT__ enough.
 ## [Unreleased]
 ### Added
 ### Changed
+- Scenario paths is no longer manually supplied to Envision server while setup. Scenario paths are automatically sent to Envision server from SMARTS during simulation startup phase.
 ### Deprecated
 ### Fixed
 ### Removed

cli/envision.py

@@ -32,13 +32,6 @@ def envision_cli():
 
 @envision_cli.command(name="start", help="Start an Envision server.")
 @click.option("-p", "--port", help="Port Envision will run on.", default=8081)
-@click.option(
-    "-s",
-    "--scenarios",
-    help="A list of directories where scenarios are stored.",
-    multiple=True,
-    default=["scenarios"],
-)
 @click.option(
     "-c",
     "--max_capacity",
@@ -49,8 +42,8 @@ def envision_cli():
     default=500,
     type=float,
 )
-def start_server(port, scenarios, max_capacity):
-    run(scenario_dirs=scenarios, max_capacity_mb=max_capacity, port=port)
+def start_server(port, max_capacity):
+    run(max_capacity_mb=max_capacity, port=port)
 
 
 envision_cli.add_command(start_server)

cli/run.py

@@ -69,8 +69,6 @@ def run_experiment(envision, envision_port, script_path, script_args):
                     "scl",
                     "envision",
                     "start",
-                    "-s",
-                    "./scenarios",
                     "-p",
                     str(envision_port),
                 ],

docs/resources/containers.rst

@@ -15,7 +15,7 @@ SMARTS docker images are hosted at `dockerhub <https://hub.docker.com/u/huaweino
     # E.g. docker run --rm -it -v $PWD:/src -p 8081:8081 huaweinoah/smarts:v0.5.1
 
     # If visualization is needed, run Envision server in the background.
-    $ scl envision start -s ./scenarios -p 8081 &
+    $ scl envision start -p 8081 &
 
     # Build the scenario. 
     # This step is required on the first time, and whenever the scenario is modified.

docs/sim/cli.rst

@@ -9,8 +9,8 @@ Examples of common usage are as follows.
 
 .. code-block:: bash
 
-   # Start envision and serve scenario assets out of ./scenarios
-   $ scl envision start --scenarios ./scenarios
+   # Start envision server
+   $ scl envision start
 
    # Build all scenarios under given directories
    $ scl scenario build-all ./scenarios

envision/client.py

@@ -76,6 +76,8 @@ def default(self, obj):
             return bool(obj)
         elif isinstance(obj, np.ndarray):
             return [self.default(x) for x in obj]
+        elif isinstance(obj, Path):
+            return str(obj)
 
         return super().default(obj)
 
@@ -292,7 +294,7 @@ def run_socket(endpoint, wait_between_retries):
 
         run_socket(endpoint, wait_between_retries)
 
-    def send(self, state: types.State):
+    def send(self, state: Union[types.State, types.Preamble]):
         """Send the given envision state to the remote as the most recent state."""
         if not self._headless and self._process.is_alive():
             self._state_queue.put(state)

envision/server.py

@@ -42,7 +42,6 @@
 from tornado.websocket import WebSocketClosedError
 
 import smarts.core.models
-from envision.types import State
 from envision.web import dist as web_dist
 from smarts.core.utils.file import path2hash
 
@@ -55,6 +54,9 @@
 # Mapping of simulation ID to the Frames data store
 FRAMES = {}
 
+# Mapping of path to map geometry files
+MAPS = {}
+
 
 class AllowCORSMixin:
     """A mixin that adds CORS headers to the page."""
@@ -318,14 +320,24 @@ def on_close(self):
     async def on_message(self, message):
         """Asynchronously receive messages from the Envision client."""
         it = ijson.parse(message)
-        frame_time = None
-        # Find the first number value, which will be the frame time.
-        for prefix, event, value in it:
-            if prefix and event == "number":
-                frame_time = float(value)
-                break
-        assert isinstance(frame_time, float)
-        self._frames.append(Frame(timestamp=frame_time, data=message))
+        next(it)  # Discard first entry: prefix="", event="start_array", value=None
+        prefix, event, value = next(it)
+        if prefix == "item" and event == "number":
+            # If the second event is a `number`, it is a payload message.
+            frame_time = float(value)
+            assert isinstance(frame_time, float)
+            self._frames.append(Frame(timestamp=frame_time, data=message))
+        elif prefix == "item" and event == "start_map":
+            # If the second event is a `start_map`, it is a preamble.
+            scenarios = [
+                value
+                for prefix, event, value in it
+                if prefix == "item.scenarios.item" and event == "string"
+            ]
+            path_map = _index_map(scenarios)
+            MAPS.update(path_map)
+        else:
+            raise tornado.web.HTTPError(400, f"Bad request message.")
 
 
 class StateWebSocket(tornado.websocket.WebSocketHandler):
@@ -346,9 +358,9 @@ def get_compression_options(self):
         return {"compression_level": 6, "mem_level": 5}
 
     async def open(self, simulation_id):
-        """Open this socket to  listen for webclient playback requests."""
+        """Open this socket to listen for webclient playback requests."""
         if simulation_id not in WEB_CLIENT_RUN_LOOPS:
-            raise tornado.web.HTTPError(404, f"Simuation `{simulation_id}` not found.")
+            raise tornado.web.HTTPError(404, f"Simulation `{simulation_id}` not found.")
 
         frequency = 10
         message_frame_volume = 100
@@ -422,20 +434,9 @@ async def serve_chunked(self, path: Path, chunk_size: int = 1024 * 1024):
 class MapFileHandler(FileHandler):
     """This handler serves map geometry to the given endpoint."""
 
-    def initialize(self, scenario_dirs: Sequence):
-        """Setup this handler. Finds and indexes all map geometry files in the given scenario
-        directories.
-        """
-        path_map = {}
-        for dir_ in scenario_dirs:
-            path_map.update(
-                {
-                    f"{path2hash(str(glb.parents[2].resolve()))}.glb": glb
-                    for glb in Path(dir_).rglob("build/map/map.glb")
-                }
-            )
-
-        super().initialize(path_map)
+    def initialize(self):
+        """Setup this handler."""
+        super().initialize(path_map=MAPS)
 
 
 class SimulationListHandler(AllowCORSMixin, tornado.web.RequestHandler):
@@ -487,7 +488,7 @@ def get(self):
             self.render(str(index_path))
 
 
-def make_app(scenario_dirs: Sequence, max_capacity_mb: float, debug: bool):
+def make_app(max_capacity_mb: float, debug: bool):
     """Create the envision web server application through composition of services."""
 
     dist_path = Path(os.path.dirname(web_dist.__file__)).absolute()
@@ -505,7 +506,6 @@ def make_app(scenario_dirs: Sequence, max_capacity_mb: float, debug: bool):
             (
                 r"/assets/maps/(.*)",
                 MapFileHandler,
-                dict(scenario_dirs=scenario_dirs),
             ),
             (r"/assets/models/(.*)", ModelFileHandler),
             (r"/(.*)", tornado.web.StaticFileHandler, dict(path=str(dist_path))),
@@ -514,20 +514,33 @@ def make_app(scenario_dirs: Sequence, max_capacity_mb: float, debug: bool):
     )
 
 
+def _index_map(scenario_dirs: Sequence[str]) -> Dict[str, Path]:
+    """Finds and indexes all map geometry files in the given scenario directories."""
+    path_map = {}
+    for dir_ in scenario_dirs:
+        path_map.update(
+            {
+                f"{path2hash(str(glb.parents[2].resolve()))}.glb": glb
+                for glb in Path(dir_).rglob("build/map/map.glb")
+            }
+        )
+
+    return path_map
+
+
 def on_shutdown():
     """Callback on shutdown of the envision server."""
     logging.debug("Shutting down Envision")
     tornado.ioloop.IOLoop.current().stop()
 
 
 def run(
-    scenario_dirs: List[str],
     max_capacity_mb: int = 500,
     port: int = 8081,
     debug: bool = False,
 ):
     """Create and run an envision web server."""
-    app = make_app(scenario_dirs, max_capacity_mb, debug=debug)
+    app = make_app(max_capacity_mb, debug=debug)
     app.listen(port)
     logging.debug("Envision listening on port=%s", port)
 
@@ -548,13 +561,6 @@ def main():
         description="The Envision server broadcasts SMARTS state to Envision web "
         "clients for visualization.",
     )
-    parser.add_argument(
-        "--scenarios",
-        help="A list of directories where scenarios are stored.",
-        default=["./scenarios"],
-        type=str,
-        nargs="+",
-    )
     parser.add_argument(
         "--port",
         help="Port Envision will run on.",
@@ -576,7 +582,6 @@ def main():
     args = parser.parse_args()
 
     run(
-        scenario_dirs=args.scenarios,
         max_capacity_mb=args.max_capacity,
         port=args.port,
         debug=args.debug,

envision/types.py

@@ -93,6 +93,13 @@ class State(NamedTuple):
     frame_time: float
 
 
+class Preamble(NamedTuple):
+    """Information for startup and synchronisation between client and server."""
+
+    scenarios: Sequence[str]
+    """Directories of simulated scenarios."""
+
+
 def format_actor_id(actor_id: str, vehicle_id: str, is_multi: bool):
     """A conversion utility to ensure that an actor id conforms to envision's actor id standard.
     Args:

examples/control/laner.py

@@ -68,7 +68,13 @@ def main(scenarios, headless, num_episodes, max_episode_steps=None):
 
     if not args.scenarios:
         args.scenarios = [
-            str(Path(__file__).absolute().parents[2] / "scenarios" / "sumo" / "loop")
+            str(Path(__file__).absolute().parents[2] / "scenarios" / "sumo" / "loop"),
+            str(
+                Path(__file__).absolute().parents[2]
+                / "scenarios"
+                / "sumo"
+                / "figure_eight"
+            ),
         ]
 
     build_scenarios(
@@ -81,4 +87,5 @@ def main(scenarios, headless, num_episodes, max_episode_steps=None):
         scenarios=args.scenarios,
         headless=args.headless,
         num_episodes=args.episodes,
+        max_episode_steps=args.max_episode_steps,
     )

examples/tools/argument_parser.py

@@ -32,6 +32,12 @@ def default_argument_parser(program: Optional[str] = None):
     parser.add_argument(
         "--headless", help="Run the simulation in headless mode.", action="store_true"
     )
+    parser.add_argument(
+        "--max_episode_steps",
+        help="Maximum number of steps to run each episode for.",
+        type=int,
+        default=100,
+    )
     parser.add_argument("--seed", type=int, default=42)
     parser.add_argument(
         "--sim-name",

smarts/core/smarts.py

@@ -29,7 +29,6 @@
 from envision import types as envision_types
 from envision.client import Client as EnvisionClient
 from smarts import VERSION
-from smarts.core.chassis import BoxChassis
 from smarts.core.plan import Plan
 from smarts.core.utils.logging import timeit
 

smarts/env/gymnasium/hiway_env_v1.py

@@ -23,6 +23,7 @@
 import os
 from enum import IntEnum
 from functools import partial
+from pathlib import Path
 from typing import (
     Any,
     Dict,
@@ -42,6 +43,7 @@
 from gymnasium.core import ActType, ObsType
 from gymnasium.envs.registration import EnvSpec
 
+from envision import types as envision_types
 from envision.client import Client as Envision
 from envision.data_formatter import EnvisionDataFormatterArgs
 from smarts.core import seed as smarts_seed
@@ -165,6 +167,7 @@ def __init__(
             )
             fixed_timestep_sec = DEFAULT_TIMESTEP
 
+        scenarios = [str(Path(scenario).resolve()) for scenario in scenarios]
         self._scenarios_iterator = Scenario.scenario_variations(
             scenarios,
             list(agent_interfaces.keys()),
@@ -177,6 +180,8 @@ def __init__(
                 headless=headless,
                 sim_name=sim_name,
             )
+            preamble = envision_types.Preamble(scenarios=scenarios)
+            visualization_client.send(preamble)
 
         self._env_renderer = None
 

smarts/env/hiway_env.py

@@ -21,10 +21,12 @@
 import logging
 import os
 import warnings
+from pathlib import Path
 from typing import Any, Dict, Optional, Sequence, Set, Tuple, Union
 
 import gym
 
+from envision import types as envision_types
 from envision.client import Client as Envision
 from envision.data_formatter import EnvisionDataFormatterArgs
 from smarts.core import seed as smarts_seed
@@ -137,6 +139,7 @@ def __init__(
             )
         self._dones_registered = 0
 
+        scenarios = [str(Path(scenario).resolve()) for scenario in scenarios]
         self._scenarios_iterator = Scenario.scenario_variations(
             scenarios,
             list(self._agent_interfaces.keys()),
@@ -154,6 +157,8 @@ def __init__(
                     "base", enable_reduction=False
                 ),
             )
+            preamble = envision_types.Preamble(scenarios=scenarios)
+            envision_client.send(preamble)
 
         self._env_renderer = None
 

smarts/env/rllib_hiway_env.py

@@ -19,10 +19,11 @@
 # THE SOFTWARE.
 import logging
 import warnings
-
+from pathlib import Path
 from ray.rllib.env.multi_agent_env import MultiAgentEnv
 
 import smarts
+from envision import types as envision_types
 from envision.client import Client as Envision
 from smarts.core.local_traffic_provider import LocalTrafficProvider
 from smarts.core.scenario import Scenario
@@ -77,8 +78,9 @@ def __init__(self, config):
         smarts.core.seed(seed + c)
 
         self._agent_specs = config["agent_specs"]
+        self._scenarios = [str(Path(scenario).resolve()) for scenario in config["scenarios"]]
         self._scenarios_iterator = Scenario.scenario_variations(
-            config["scenarios"],
+            self._scenarios,
             list(self._agent_specs.keys()),
         )
 
@@ -206,6 +208,8 @@ def _build_smarts(self):
                 output_dir=self._envision_record_data_replay_path,
                 headless=self._headless,
             )
+            preamble = envision_types.Preamble(scenarios=self._scenarios)
+            envision.send(preamble)
 
         sumo_traffic = SumoTrafficSimulation(
             headless=self._sumo_headless,