From 1c433e6f3a392a81ba0b961ba1465195907b4437 Mon Sep 17 00:00:00 2001 From: Wolfgang Hess Date: Tue, 10 Jan 2017 10:43:16 +0100 Subject: [PATCH] Refactor the documentation update script. (#179) --- .../proto/sparse_pose_graph_options.proto | 2 +- ...ast_correlative_scan_matcher_options.proto | 2 +- .../proto/adaptive_voxel_filter_options.proto | 2 +- docs/source/configuration.rst | 42 ++-- scripts/update_configuration_doc.py | 181 +++++++++++------- 5 files changed, 140 insertions(+), 89 deletions(-) diff --git a/cartographer/mapping/proto/sparse_pose_graph_options.proto b/cartographer/mapping/proto/sparse_pose_graph_options.proto index 4812bf0ad2..7436231d49 100644 --- a/cartographer/mapping/proto/sparse_pose_graph_options.proto +++ b/cartographer/mapping/proto/sparse_pose_graph_options.proto @@ -39,4 +39,4 @@ message SparsePoseGraphOptions { // Rate at which we sample a single trajectory's scans for global // localization. optional double global_sampling_ratio = 5; -}; +} diff --git a/cartographer/mapping_2d/scan_matching/proto/fast_correlative_scan_matcher_options.proto b/cartographer/mapping_2d/scan_matching/proto/fast_correlative_scan_matcher_options.proto index b28ad1a7f5..5d57170ed4 100644 --- a/cartographer/mapping_2d/scan_matching/proto/fast_correlative_scan_matcher_options.proto +++ b/cartographer/mapping_2d/scan_matching/proto/fast_correlative_scan_matcher_options.proto @@ -27,4 +27,4 @@ message FastCorrelativeScanMatcherOptions { // Number of precomputed grids to use. optional int32 branch_and_bound_depth = 2; -}; +} diff --git a/cartographer/sensor/proto/adaptive_voxel_filter_options.proto b/cartographer/sensor/proto/adaptive_voxel_filter_options.proto index a6dcaeaf6b..935f8c5500 100644 --- a/cartographer/sensor/proto/adaptive_voxel_filter_options.proto +++ b/cartographer/sensor/proto/adaptive_voxel_filter_options.proto @@ -26,4 +26,4 @@ message AdaptiveVoxelFilterOptions { // Points further away from the origin are removed. optional float max_range = 3; -}; +} diff --git a/docs/source/configuration.rst b/docs/source/configuration.rst index cde6d63e7b..9c4d3e91ea 100644 --- a/docs/source/configuration.rst +++ b/docs/source/configuration.rst @@ -20,7 +20,7 @@ Configuration .. needed and run scripts/update_configuration_doc.py. cartographer.common.proto.CeresSolverOptions --------------------------------------------- +============================================ bool use_nonmonotonic_steps Configure the Ceres solver. See the Ceres documentation for more @@ -34,7 +34,7 @@ int32 num_threads cartographer.kalman_filter.proto.PoseTrackerOptions ---------------------------------------------------- +=================================================== double position_model_variance Model variances depend linearly on time. @@ -57,7 +57,7 @@ int32 num_odometry_states cartographer.mapping.proto.MapBuilderOptions --------------------------------------------- +============================================ bool use_trajectory_builder_2d Not yet documented. @@ -79,7 +79,7 @@ SparsePoseGraphOptions sparse_pose_graph_options cartographer.mapping.proto.SparsePoseGraphOptions -------------------------------------------------- +================================================= int32 optimize_every_n_scans Online loop closure: If positive, will run the loop closure while the map @@ -101,7 +101,7 @@ double global_sampling_ratio cartographer.mapping.sparse_pose_graph.proto.ConstraintBuilderOptions ---------------------------------------------------------------------- +===================================================================== double sampling_ratio A constraint will be added if the proportion of added constraints to @@ -140,7 +140,7 @@ mapping_3d.scan_matching.proto.CeresScanMatcherOptions ceres_scan_matcher_option cartographer.mapping.sparse_pose_graph.proto.OptimizationProblemOptions ------------------------------------------------------------------------ +======================================================================= double huber_scale Scaling parameter for Huber loss function. @@ -165,7 +165,7 @@ common.proto.CeresSolverOptions ceres_solver_options cartographer.mapping_2d.proto.LaserFanInserterOptions ------------------------------------------------------ +===================================================== double hit_probability Probability change for a hit (this will be converted to odds and therefore @@ -181,7 +181,7 @@ bool insert_free_space cartographer.mapping_2d.proto.LocalTrajectoryBuilderOptions ------------------------------------------------------------ +=========================================================== float laser_min_range Laser returns outside these ranges will be dropped. @@ -238,7 +238,7 @@ bool use_imu_data cartographer.mapping_2d.proto.SubmapsOptions --------------------------------------------- +============================================ double resolution Resolution of the map in meters. @@ -259,7 +259,7 @@ LaserFanInserterOptions laser_fan_inserter_options cartographer.mapping_2d.scan_matching.proto.CeresScanMatcherOptions -------------------------------------------------------------------- +=================================================================== double occupied_space_weight Scaling parameters for each cost functor. @@ -279,7 +279,7 @@ common.proto.CeresSolverOptions ceres_solver_options cartographer.mapping_2d.scan_matching.proto.FastCorrelativeScanMatcherOptions ------------------------------------------------------------------------------ +============================================================================= double linear_search_window Minimum linear search window in which the best possible scan alignment @@ -294,7 +294,7 @@ int32 branch_and_bound_depth cartographer.mapping_2d.scan_matching.proto.RealTimeCorrelativeScanMatcherOptions ---------------------------------------------------------------------------------- +================================================================================= double linear_search_window Minimum linear search window in which the best possible scan alignment @@ -312,7 +312,7 @@ double rotation_delta_cost_weight cartographer.mapping_3d.proto.KalmanLocalTrajectoryBuilderOptions ------------------------------------------------------------------ +================================================================= bool use_online_correlative_scan_matching Whether to solve the online scan matching first using the correlative scan @@ -332,7 +332,7 @@ double odometer_rotational_variance cartographer.mapping_3d.proto.LaserFanInserterOptions ------------------------------------------------------ +===================================================== double hit_probability Probability change for a hit (this will be converted to odds and therefore @@ -348,11 +348,11 @@ int32 num_free_space_voxels cartographer.mapping_3d.proto.LocalTrajectoryBuilderOptions ------------------------------------------------------------ +=========================================================== cartographer.mapping_3d.proto.MotionFilterOptions -------------------------------------------------- +================================================= double max_time_seconds Threshold above which a new scan is inserted based on time. @@ -365,7 +365,7 @@ double max_angle_radians cartographer.mapping_3d.proto.OptimizingLocalTrajectoryBuilderOptions ---------------------------------------------------------------------- +===================================================================== double high_resolution_grid_weight Not yet documented. @@ -390,7 +390,7 @@ double odometry_rotation_weight cartographer.mapping_3d.proto.SubmapsOptions --------------------------------------------- +============================================ double high_resolution Resolution of the 'high_resolution' map in meters used for local SLAM and @@ -414,7 +414,7 @@ LaserFanInserterOptions laser_fan_inserter_options cartographer.mapping_3d.scan_matching.proto.CeresScanMatcherOptions -------------------------------------------------------------------- +=================================================================== double translation_weight Scaling parameters for each cost functor. @@ -434,7 +434,7 @@ common.proto.CeresSolverOptions ceres_solver_options cartographer.mapping_3d.scan_matching.proto.FastCorrelativeScanMatcherOptions ------------------------------------------------------------------------------ +============================================================================= int32 branch_and_bound_depth Number of precomputed grids to use. @@ -463,7 +463,7 @@ double angular_search_window cartographer.sensor.proto.AdaptiveVoxelFilterOptions ----------------------------------------------------- +==================================================== float max_length 'max_length' of a voxel edge. diff --git a/scripts/update_configuration_doc.py b/scripts/update_configuration_doc.py index c20de1298d..03a460a5f7 100755 --- a/scripts/update_configuration_doc.py +++ b/scripts/update_configuration_doc.py @@ -47,80 +47,131 @@ """ NODOC = 'Not yet documented.' -def GenerateDocumentation(output_dict, proto_file_name): - copyright = True - message = None - content = []; - package = None - multiline = None - print("Reading '%s'..." % proto_file_name) - for line in io.open(proto_file_name, encoding='UTF-8'): + +def ForEachProtoFile(root, callback): + for dirpath, dirnames, filenames in os.walk(root): + for name in filenames: + if name.endswith('.proto'): + path = os.path.join(dirpath, name) + print("Found '%s'..." % path) + assert not os.path.islink(path) + callback(io.open(path, encoding='UTF-8')) + + +class Message(object): + def __init__(self, name, preceding_comments): + self.name = name + self.preceding_comments = preceding_comments + self.trailing_comments = None + self.options = [] + + def AddTrailingComments(self, comments): + self.trailing_comments = comments + + def AddOption(self, name, comments): + self.options.append((name, comments)) + + +def ParseProtoFile(proto_file): + """Computes the list of Message objects of the option messages in a file.""" + line_iter = iter(proto_file) + + # We ignore the license header and search for the 'package' line. + for line in line_iter: line = line.strip() - if copyright: - if not line.startswith('//'): - copyright = False - continue - if package is None: - if line.startswith('package'): - assert line[-1] == ';' - package = line[7:-1].strip() - continue - if line.startswith('//'): - content_line = line[2:].strip() - if not content_line.startswith('NEXT ID:'): - content.append(content_line) - continue - if message is None: - if line.startswith('message') and line.endswith('Options {'): - message = package + '.' + line[7:-1].strip() - print(" Found '%s'." % message) - assert message not in output_dict - message_list = [message, '=' * len(message), ''] - output_dict[message] = message_list - message_list.extend(content) - content = [] - continue - elif line.endswith('}'): - message_list.extend(content) - content = [] - message_list = None - message = None + if line.startswith('package'): + assert line[-1] == ';' + package = line[7:-1].strip() + break else: - assert not line.startswith('required') - if multiline is None: - if line.startswith('optional'): - multiline = line + assert '}' not in line + + message_list = [] + while True: + # Search for the next options message and capture preceding comments. + message_comments = [] + for line in line_iter: + line = line.strip() + if '}' in line: + # The preceding comments were for a different message it seems. + message_comments = [] + elif line.startswith('//'): + # We keep comments preceding an options message. + comment = line[2:].strip() + if not comment.startswith('NEXT ID:'): + message_comments.append(comment) + elif line.startswith('message') and line.endswith('Options {'): + message_name = package + '.' + line[7:-1].strip() + break + else: + # We reached the end of file. + break + print(" Found '%s'." % message_name) + message = Message(message_name, message_comments) + message_list.append(message) + + # We capture the contents of this message. + option_comments = [] + multiline = None + for line in line_iter: + line = line.strip() + if '}' in line: + # We reached the end of this message. + message.AddTrailingComments(option_comments) + break + elif line.startswith('//'): + comment = line[2:].strip() + if not comment.startswith('NEXT ID:'): + option_comments.append(comment) + else: + assert not line.startswith('required') + if multiline is None: + if line.startswith('optional'): + multiline = line + else: + continue else: + multiline += ' ' + line + if not multiline.endswith(';'): continue - else: - multiline += ' ' + line - if not multiline.endswith(';'): - continue - assert len(multiline) < 200 - option = multiline[8:-1].strip().rstrip('0123456789').strip() - assert option.endswith('=') - option = option[:-1].strip(); - print(" Option '%s'." % option) - multiline = None - message_list.append(option) - if len(content) == 0: - content.append(NODOC) - for option_description_line in content: - message_list.append(' ' + option_description_line) - content = [] - message_list.append('') + assert len(multiline) < 200 + option_name = multiline[8:-1].strip().rstrip('0123456789').strip() + assert option_name.endswith('=') + option_name = option_name[:-1].strip(); + print(" Option '%s'." % option_name) + multiline = None + message.AddOption(option_name, option_comments) + option_comments = [] + + return message_list + + +def GenerateDocumentation(output_dict, proto_file): + for message in ParseProtoFile(proto_file): + content = [message.name, '=' * len(message.name), ''] + assert message.name not in output_dict + output_dict[message.name] = content + if message.preceding_comments: + content.extend(preceding_comments) + content.append('') + for option_name, option_comments in message.options: + content.append(option_name) + if not option_comments: + option_comments.append(NODOC) + for comment in option_comments: + content.append(' ' + comment) + content.append('') + if message.trailing_comments: + content.extend(message.trailing_comments) + content.append('') def GenerateDocumentationRecursively(output_file, root): """Recursively generates documentation, sorts and writes it.""" output_dict = {} - for root, dirs, files in os.walk(root): - for name in files: - if name.endswith('.proto'): - path = os.path.join(root, name) - assert not os.path.islink(path) - GenerateDocumentation(output_dict, path) - + def callback(proto_file): + GenerateDocumentation(output_dict, proto_file) + ForEachProtoFile(root, callback) output = ['\n'.join(doc) for key, doc in sorted(list(output_dict.items()))] print('\n\n'.join(output), file=output_file)