[CR-1231] Bump to 0.1.8a1

In summary:

* convert_rosbag2: Add new flexible format for parsing conversion config
  files
* convert_rosbag2: Enable saving compressed PNG / JPEG images

-----

In more details:

Includes the following changes to the `convert_rosbag2` script:

* Introduce a new "flexible" (contrary to the pre-existing rigid) format
  for specifying the topics and Slamcore subdirectories for conversion.

  The new version allows conversion of rosbags using a config file like
  the following:

  ```jsonc
  {
    // conversion metadata
    "conversion_meta": {
      "format": "flexible",
      "version": 1
    },

    // SLAM Input topics
    "cam0": {
      "topics": ["/slamcore/visible_0/image_raw", "/slamcore/ir_0/image_raw"]
    },
    "cam1": {
      "topics": ["/slamcore/visible_1/image_raw", "/slamcore/ir_1/image_raw"]
    },
    "imu0": {
      "topic": "/irrelevant_imu"
    },
    "odometry0": {
      "topic": "/slamcore/odom",
      "required": false
    },
    "groundtruth0": {
      "topic": "/irrelevant_gts"
    }
  }
  ```

  The above will look for a topic named `/slamcore/visible_0/image_raw`
  and if it can't find it it will fall back to the second specified
  topic name, in the `cam0` case (`/slamcore/ir_0/image_raw`). It will
  give an error if it can't match any of the specified topics, unless
  the user has specified the `required: false` key.

* Introduce a new `--disable-progress-bar` CLI flag and app argument. This
  way, the script won't produce unwanted noisy standard output/error
  when executed automatically (as we do in `dataset-analysis`). This
  also sometimes led to errors uploading reports to Slack as the text
  report was too big to be uploaded as a code snippet.

* Introduce a convert_rosbag2 configuration struct - `Config` which
  captures all the configuration variables that the script needs. This
  allows building this struct and calling the `convert_rosbag2` function
  without having to call it via a subprocess in `dataset-analysis`

* Only convert topics if they contain at least one message. Otherwise
  this may lead to errors. E.g., during trajectory alighnment, if
  there's an optimized trajectory nav_path topic, we'll use that as the
  optimized trajectory. However, the GT alignment will fail if the
  opitmized trajectory data.csv file is empty. In that case we should
  altogether have discarded that topic in the first place

Also includes:

Fix small typo in var name -> `verbotsity_to_logging_lvl` -> `verbosity_to_logging_lvl`

Also includes:

* Solve progress bar race condition

  In case where the rosbag we're converting is short, we may end up
  joining the Image queue faster than the time it takes for the progress
  bar to update the standard output. In that case the program exits but
  daemonic thread on which we call `progress_bar.update()` is still
  active and will attempt to access a progress_bar instance that has
  gone out of scope. To solve that, we're calling
  `progress_bar.update()` before releasing the imq lock (via
  `imgq.task_done()`)

.github/workflows/ci.yml

@@ -57,7 +57,8 @@ jobs:
 
       - name: Setup Python Poetry
         uses: abatilo/actions-poetry@v2.1.3
-
+        with:
+          poetry-version: 1.6.1
       - name: Install prerequisites
         run: "poetry version && poetry install --extras ros2"
       - name: Run tests
@@ -85,6 +86,8 @@ jobs:
 
       - name: Setup Python Poetry
         uses: abatilo/actions-poetry@v2.1.3
+        with:
+          poetry-version: 1.6.1
       - name: Install prerequisites
         run: poetry install --extras ros2
       - name: Check style
@@ -111,6 +114,8 @@ jobs:
 
       - name: Setup Python Poetry
         uses: abatilo/actions-poetry@v2.1.3
+        with:
+          poetry-version: 1.6.1
       - name: Install prerequisites
         run: poetry install --extras ros2
       - name: Lint code

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "slamcore_utils"
-version = "0.1.8a"
+version = "0.1.8a1"
 description = "SLAMcore SLAM Utilities"
 authors = ["Nikos Koukis <nikolaos@slamcore.com>"]
 license = "BSD License"

slamcore_utils/arg_parser.py

@@ -1,8 +1,15 @@
-"""Helper methods to utilize to use with the argparse module for command line options parsing. """
-
-from argparse import ArgumentParser, ArgumentTypeError
+"""
+Helper methods to utilize to use with the argparse module for command line options parsing.
+"""
+
+from argparse import (
+    ArgumentDefaultsHelpFormatter,
+    ArgumentParser,
+    ArgumentTypeError,
+    RawDescriptionHelpFormatter,
+)
 from pathlib import Path
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 help_msgs: Dict[str, str] = {}
 arg_defaults: Dict[str, Any] = {}
@@ -49,7 +56,10 @@ def non_existing_dir(path: str) -> Path:
 
 
 def add_bool_argument(
-    parser: ArgumentParser, arg_name: str, default: bool = None, true_help: str = None
+    parser: ArgumentParser,
+    arg_name: str,
+    default: Optional[bool] = None,
+    true_help: Optional[str] = None,
 ):
     """Add a boolean CLI argument to the given ArgumentParser object.
 
@@ -103,11 +113,23 @@ def _format(s: str):
 
     group = parser.add_mutually_exclusive_group()
 
-    true_help = true_help if default == False else f"{true_help} [default]"
-    false_help = "" if default == True else "[default]"
+    true_help = true_help if default is False else f"{true_help} [default]"
+    false_help = "" if default is True else "[default]"
 
     group.add_argument(_format("--"), dest=arg_name, action="store_true", help=true_help)
     group.add_argument(_format("--no-"), dest=arg_name, action="store_false", help=false_help)
 
     if default:
         eval("group.set_defaults({}=default)".format(arg_name))
+
+
+# Argument Help Formatter ---------------------------------------------------------------------
+class ArgumentDefaultsHelpAndRawFormatter(
+    RawDescriptionHelpFormatter, ArgumentDefaultsHelpFormatter
+):
+    """
+    Parser that displays both the help description in its raw format and also shows the
+    default options for its arguments.
+    """
+
+    pass

slamcore_utils/dataset_subdir_writer.py

@@ -24,6 +24,7 @@
 import abc
 from pathlib import Path
 from typing import Type
+from slamcore_utils.logging import logger as pkg_logger
 
 
 class DatasetSubdirWriter(abc.ABC):
@@ -33,9 +34,10 @@ class DatasetSubdirWriter(abc.ABC):
     Slamcore output dataset directory.
     """
 
-    def __init__(self, directory: Path):
+    def __init__(self, directory: Path, logger=pkg_logger):
         self.directory = directory
         self.directory.mkdir(parents=True, exist_ok=False)
+        self.logger = logger
 
     def register_ros_msg_type(self, msg_type: Type) -> None:
         """

slamcore_utils/logging.py

@@ -58,7 +58,7 @@ def _setup_logging(name: str, format_: str, level: int):
 }
 
 
-def verbotsity_to_logging_lvl(verbosity: int) -> int:
+def verbosity_to_logging_lvl(verbosity: int) -> int:
     """Map CLI-provided verbosity count e.g., -vvv to logging module level."""
     if verbosity > 2:
         return logging.DEBUG

slamcore_utils/progress_bar.py

@@ -98,3 +98,14 @@ def write(cls, s: str, end: str = "\n", *args, **kargs):
     progress_bar = _ProgressBar
 
 progress_bar = cast(Type[_ProgressBar], progress_bar)
+
+class DummyProgressBar:
+    """Progress bar class that does nothing."""
+    def __init__(self, *args, **kargs):
+        pass
+
+    def __iter__(self) -> Any:
+        pass
+
+    def update(self, *args, **kargs) -> Any:
+        pass

slamcore_utils/ros2/__init__.py

@@ -22,6 +22,11 @@
 
 
 try:
+    from .plugin_utils import (
+        get_internal_plugins_dir,
+        load_converter_plugins,
+        load_converter_plugins_from_multiple_files,
+    )
     from .ros2_converter_plugin import (
         Ros2ConverterPlugin,
         Ros2PluginInitializationFailureError,
@@ -33,13 +38,19 @@
         init_rosbag2_reader_handle_exceptions,
     )
 
+    from .conversion_format import ConversionFormat
+
     __all__ = [
         "Ros2ConverterPlugin",
         "Ros2PluginInitializationFailureError",
         "get_topic_names_to_message_counts",
         "get_topic_names_to_types",
         "init_rosbag2_reader",
         "init_rosbag2_reader_handle_exceptions",
+        "get_internal_plugins_dir",
+        "load_converter_plugins",
+        "load_converter_plugins_from_multiple_files",
+        "ConversionFormat",
     ]
 except ModuleNotFoundError as e:
     print(

slamcore_utils/ros2/conversion_format.py

@@ -0,0 +1,61 @@
+from enum import Enum, auto
+from typing import Any, Dict, Tuple
+from slamcore_utils.logging import logger
+
+
+class ConversionFormat(Enum):
+    """Conversion formats supported.
+
+    These define the way we parse the conversion configuration file.
+    """
+
+    RIGID = auto()
+    FLEXIBLE = auto()
+
+    @staticmethod
+    def from_name(name: str) -> "ConversionFormat":
+        return _names_to_conversion_formats[name]
+
+    def __str__(self) -> str:
+        return self.name.lower()
+
+    def __repr__(self) -> str:
+        return f"{self.__class__.__name__}.{self.name}"
+
+    @classmethod
+    def detect_format(cls, json: Dict[str, Any]) -> Tuple["ConversionFormat", int]:
+        """
+        Return a tuple of (conversion_format, version_format) by parsing the json contents
+        provided. Always resort to the rigid format in case of errors.
+
+        Do remove the conversion meta section from the JSON contents at the end of this call.
+        """
+        conversion_meta_section = "conversion_meta"
+        meta_section = json.pop(conversion_meta_section, None)
+
+        if meta_section is None:
+            logger.warning(
+                f"No {conversion_meta_section} section. "
+                f"Resorting to {ConversionFormat.RIGID} conversion format ..."
+            )
+            return (ConversionFormat.RIGID, 0)
+
+        try:
+            format_str = meta_section["format"]
+            version_int = int(meta_section["version"])
+        except Exception as e:
+            raise RuntimeError(
+                f"Invalid {conversion_meta_section} section. Cannot continue."
+            ) from e
+
+        try:
+            format_and_version = cls.from_name(format_str), version_int
+        except KeyError as e:
+            raise RuntimeError(f"Unknown conversion format -> {format_str}.") from e
+
+        logger.info(f"Determined conversion format/version: {format_and_version} .")
+        return format_and_version
+
+
+_conversion_formats_to_names = {cf: str(cf) for cf in ConversionFormat}
+_names_to_conversion_formats = {v: k for k, v in _conversion_formats_to_names.items()}

slamcore_utils/ros2/plugin_utils.py

@@ -0,0 +1,98 @@
+from functools import reduce
+from pathlib import Path
+from runpy import run_path
+from slamcore_utils.logging import logger
+from typing import Sequence, cast
+import operator
+import pkg_resources
+
+from .ros2_converter_plugin import Ros2ConverterPlugin
+
+
+def get_internal_plugins_dir() -> Path:
+    """
+    Get the path to the internal ROS 2 plugins.
+
+    This depends on whether the user has installed this package and executed the installed
+    script or whether they've directly executed the script via `python3 -m`
+    """
+    if __package__:
+        return (
+            Path(pkg_resources.resource_filename(__package__.split(".")[0], "ros2"))
+            / "ros2_converter_plugins"
+        )
+    else:
+        return Path(__file__).absolute().parent / "ros2_converter_plugins"
+
+
+def load_converter_plugins_from_multiple_files(
+    converter_plugin_paths: Sequence[Path],
+    raise_on_error: bool = True,
+) -> Sequence[Ros2ConverterPlugin]:
+    if converter_plugin_paths:
+        converter_plugins = cast(
+            Sequence[Ros2ConverterPlugin],
+            reduce(
+                operator.concat,
+                (
+                    load_converter_plugins(plugin_path, raise_on_error=raise_on_error)
+                    for plugin_path in converter_plugin_paths
+                ),
+            ),
+        )
+    else:
+        return []
+
+    # Sanity check, each one of converter_plugins var is of the right type
+    for cp in converter_plugins:
+        if not isinstance(cp, Ros2ConverterPlugin):
+            raise RuntimeError(
+                "One of the specified converter plugins is not of type "
+                "Ros2ConverterPlugin, cannot proceed"
+            )
+
+    return converter_plugins
+
+
+def load_converter_plugins(
+    plugin_path: Path, raise_on_error: bool
+) -> Sequence[Ros2ConverterPlugin]:
+    """Load all the ROS 2 Converter Plugins specified in the given plugin python module.
+
+    In case of errors print the right error messages acconrdingly.
+    """
+    logger.debug(f"Loading ROS 2 converter plugin from {plugin_path} ...")
+    try:
+        ns_dict = run_path(str(plugin_path))
+    except BaseException as e:
+        e_str = (
+            "Failed to load ROS 2 converter plugin from "
+            f"{plugin_path.relative_to(plugin_path.parent)}\n\nError: {e}\n\n"
+        )
+        if raise_on_error is True:
+            raise RuntimeError(f"{e_str}Exiting ...") from e
+        else:
+            logger.debug(e_str)
+            return []
+
+    # Sanity check, specified converter_plugins var
+    if "converter_plugins" not in ns_dict:
+        raise RuntimeError(
+            f"No converter plugins were exported in specified plugin -> {plugin_path}\n",
+            (
+                'Make sure that you have initialized a variable named "converter_plugins" '
+                "at the top-level of the said plugin."
+            ),
+        )
+
+    converter_plugins = ns_dict["converter_plugins"]
+
+    # Sanity check, converter_plugins var is of the right type
+    if not isinstance(converter_plugins, Sequence):
+        raise RuntimeError(
+            f"Found the converter_plugins at the top-level of the plugin ({plugin_path}) "
+            "but that variable is not of type Sequence. Its value is "
+            f"{converter_plugins} and of type {type(converter_plugins)}"
+        )
+
+    return converter_plugins

slamcore_utils/ros2/ros2_converter_plugins/gnss.py

@@ -27,11 +27,10 @@
 
 import numpy as np
 
-from slamcore_utils import DatasetSubdirWriter, MeasurementType, setup_pkg_logging
+from slamcore_utils import DatasetSubdirWriter, MeasurementType
 from slamcore_utils.ros2 import Ros2ConverterPlugin, Ros2PluginInitializationFailureError
 
 plugin_name = Path(__file__).name
-logger = setup_pkg_logging(plugin_name)
 
 try:
     from gps_msgs.msg import GPSFix
@@ -54,9 +53,6 @@ class GPSAsPoseStampedWriter(DatasetSubdirWriter):
     EARTH_RADIUS_M = 6378137.0
     FLATTENING_RATIO = 1.0 / 298.257224
 
-    def __init__(self, directory):
-        super().__init__(directory=directory)
-
     def prepare_write(self) -> None:
         self.ofs_data = (self.directory / "data.csv").open("w", newline="")
         self.csv_writer = csv.writer(self.ofs_data, delimiter=",")

slamcore_utils/ros2/ros2_converter_plugins/nav_path.py

@@ -24,13 +24,12 @@
 import csv
 from pathlib import Path
 from geometry_msgs.msg import PoseStamped
+from slamcore_utils.dataset_subdir_writer import DatasetSubdirWriter
+from slamcore_utils.measurement_type import MeasurementType
 
-from slamcore_utils import DatasetSubdirWriter, MeasurementType, setup_pkg_logging
 from slamcore_utils.ros2 import Ros2ConverterPlugin, Ros2PluginInitializationFailureError
 
 plugin_name = Path(__file__).name
-logger = setup_pkg_logging(plugin_name)
-
 try:
     from nav_msgs.msg import Path as NavPath
 except ModuleNotFoundError:
@@ -48,9 +47,6 @@
 class NavPathWriter(DatasetSubdirWriter):
     """Convert a single nav_msgs/Path msg to a Slamcore-compatible CSV file of poses."""
 
-    def __init__(self, directory):
-        super().__init__(directory=directory)
-
     def prepare_write(self) -> None:
         self.ofs_data = (self.directory / "data.csv").open("w", newline="")
         self.csv_writer = csv.writer(self.ofs_data, delimiter=",")
@@ -89,7 +85,7 @@ def write(self, msg: NavPath) -> None:
         self.write = self._write_nop
 
     def _write_nop(self, msg) -> None:
-        logger.warning(
+        self.logger.warning(
             f"{self.__class__.__name__} expected only a single message to convert. "
             f"Instead it just received additional message(s) - msg: {msg}"
         )