Extra packages implementation (#12)

* Add extra packages step

* Add extra packages to CI testing

* Debug CI

* Add vcs install to CI

* Fix typo

* Avoid vcs call

* Add yaml depend

* Fix yaml depend

* Move import on main script

* Print copied extra packages

* Improve method name

ci/extra_packages/extra_packages.repos

@@ -0,0 +1,5 @@
+repositories:
+  control_msgs:
+    type: git
+    url: https://github.com/ros-controls/control_msgs
+    version: galactic-devel

ci/extra_packages/my_custom_message/CMakeLists.txt

@@ -0,0 +1,41 @@
+cmake_minimum_required(VERSION 3.5)
+project(my_custom_message)
+
+# Default to C99
+if(NOT CMAKE_C_STANDARD)
+  set(CMAKE_C_STANDARD 99)
+endif()
+
+# Default to C++14
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 14)
+endif()
+
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic)
+endif()
+
+# find dependencies
+find_package(ament_cmake REQUIRED)
+# uncomment the following section in order to fill in
+# further dependencies manually.
+# find_package(<dependency> REQUIRED)
+
+if(BUILD_TESTING)
+  find_package(ament_lint_auto REQUIRED)
+  # the following line skips the linter which checks for copyrights
+  # uncomment the line when a copyright and license is not present in all source files
+  #set(ament_cmake_copyright_FOUND TRUE)
+  # the following line skips cpplint (only works in a git repo)
+  # uncomment the line when this package is not in a git repo
+  #set(ament_cmake_cpplint_FOUND TRUE)
+  ament_lint_auto_find_test_dependencies()
+endif()
+
+find_package(rosidl_default_generators REQUIRED)
+
+rosidl_generate_interfaces(${PROJECT_NAME}
+  "msg/MyCustomMessage.msg"
+ )
+
+ament_package()

ci/extra_packages/my_custom_message/msg/MyCustomMessage.msg

@@ -0,0 +1,13 @@
+bool bool_test
+byte byte_test
+char char_test
+float32 float32_test
+float64 double_test
+int8 int8_test
+uint8 uint8_test
+int16 int16_test
+uint16 uint16_test
+int32 int32_test
+uint32 uint32_test
+int64 int64_test
+uint64 uint64_test

ci/extra_packages/my_custom_message/package.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<?xml-model href="http://download.ros.org/schema/package_format3.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<package format="3">
+  <name>my_custom_message</name>
+  <version>0.0.0</version>
+  <description>TODO: Package description</description>
+  <maintainer email="root@todo.todo">root</maintainer>
+  <license>TODO: License declaration</license>
+
+  <buildtool_depend>ament_cmake</buildtool_depend>
+
+  <build_depend>rosidl_default_generators</build_depend>
+  <exec_depend>rosidl_default_runtime</exec_depend>
+  <member_of_group>rosidl_interface_packages</member_of_group>
+
+  <test_depend>ament_lint_auto</test_depend>
+  <test_depend>ament_lint_common</test_depend>
+
+  <export>
+    <build_type>ament_cmake</build_type>
+  </export>
+</package>

ci/src/main.cpp

@@ -14,6 +14,12 @@
 
 #include <std_msgs/msg/int32.h>
 
+// Test extra packages
+#include <control_msgs/msg/joint_controller_state.h>
+#include <my_custom_message/msg/my_custom_message.h>
+control_msgs__msg__JointControllerState control_message;
+my_custom_message__msg__MyCustomMessage custom_msg;
+
 rcl_publisher_t publisher;
 std_msgs__msg__Int32 msg;
 rclc_executor_t executor;

extra_script.py

@@ -1,20 +1,21 @@
 Import("env")
 import os, sys
-import microros_utils.library_builder as library_builder
 
 ##############################
 #### Install dependencies ####
 ##############################
 
 pip_packages = [x.split("==")[0] for x in os.popen('{} -m pip freeze'.format(env['PYTHONEXE'])).read().split('\n')]
-required_packages = ["catkin-pkg", "lark-parser", "empy", "colcon-common-extensions", "importlib-resources"]
+required_packages = ["catkin-pkg", "lark-parser", "empy", "colcon-common-extensions", "importlib-resources", "pyyaml"]
 if all([x in pip_packages for x in required_packages]):
     print("All required Python pip packages are installed")
 
 for p in [x for x in required_packages if x not in pip_packages]:
     print('Installing {} with pip at PlatformIO environment'.format(p))
     env.Execute('$PYTHONEXE -m pip install {}'.format(p))
 
+import microros_utils.library_builder as library_builder
+
 ##########################
 #### Global variables ####
 ##########################
@@ -39,6 +40,7 @@
 board = env['BOARD']
 framework = env['PIOFRAMEWORK'][0]
 main_path = os.path.realpath(".")
+extra_packages_path = "{}/extra_packages".format(env['PROJECT_DIR'])
 
 selected_board_meta = boards_metas[board] if board in boards_metas else "colcon.meta"
 
@@ -70,7 +72,7 @@
     "{} {} -fno-rtti -DCLOCK_MONOTONIC=0 -D'__attribute__(x)='".format(' '.join(env['CXXFLAGS']), ' '.join(env['CCFLAGS']))
 )
 
-builder = library_builder.Build(main_path)
+builder = library_builder.Build(library_folder=main_path, packages_folder=extra_packages_path)
 builder.run('{}/metas/{}'.format(main_path, selected_board_meta), cmake_toolchain.path, microros_user_meta)
 
 #######################################################

microros_utils/library_builder.py

@@ -1,5 +1,6 @@
 import os
 import json
+import yaml
 import shutil
 import xml.etree.ElementTree as xml_parser
 
@@ -127,8 +128,9 @@ class Build:
         'galactic': ['rcl_logging_log4cxx', 'rcl_logging_spdlog', 'rcl_yaml_param_parser', 'rclc_examples']
     }
 
-    def __init__(self, library_folder, distro = 'galactic'):
+    def __init__(self, library_folder, packages_folder, distro = 'galactic'):
         self.library_folder = library_folder
+        self.packages_folder = packages_folder
         self.build_folder = library_folder + "/build"
         self.distro = distro
 
@@ -195,6 +197,56 @@ def download_mcu_environment(self):
 
                 print('\t - Downloaded {}{}'.format(package.name, " (ignored)" if package.ignored else ""))
 
+        self.download_extra_packages()
+
+    def download_extra_packages(self):
+        if not os.path.exists(self.packages_folder):
+            print("\t - Extra packages folder not found, skipping...")
+            return
+
+        print("Checking extra packages")
+
+        # Load and clone repositories from extra_packages.repos file
+        extra_repos = self.get_repositories_from_yaml("{}/extra_packages.repos".format(self.packages_folder))
+        for repo_name in extra_repos:
+            repo_values = extra_repos[repo_name]
+            version = repo_values['version'] if 'version' in repo_values else None
+            Repository(repo_name, repo_values['url'], self.distro, version).clone(self.mcu_src_folder)
+            print("\t - Downloaded {}".format(repo_name))
+
+        extra_folders = os.listdir(self.packages_folder)
+        if 'extra_packages.repos' in extra_folders:
+            extra_folders.remove('extra_packages.repos')
+
+        for folder in extra_folders:
+            print("\t - Adding {}".format(folder))
+
+        shutil.copytree(self.packages_folder, self.mcu_src_folder, ignore=shutil.ignore_patterns('extra_packages.repos'), dirs_exist_ok=True)
+
+    def get_repositories_from_yaml(self, yaml_file):
+        repos = {}
+        try:
+            with open(yaml_file, 'r') as repos_file:
+                root = yaml.safe_load(repos_file)
+                repositories = root['repositories']
+
+            if repositories:
+                for path in repositories:
+                    repo = {}
+                    attributes = repositories[path]
+                    try:
+                        repo['type'] = attributes['type']
+                        repo['url'] = attributes['url']
+                        if 'version' in attributes:
+                            repo['version'] = attributes['version']
+                    except KeyError as e:
+                        continue
+                    repos[path] = repo
+        except (yaml.YAMLError, KeyError, TypeError) as e:
+            print("Error on {}: {}".format(yaml_file, e))
+        finally:
+            return repos
+
     def build_mcu_environment(self, meta_file, toolchain_file, user_meta = ""):
         if os.path.exists(self.mcu_folder + '/build'):
             print("micro-ROS already built")