[docs] generate parameters description from config file. Stage 1 (#1409)

* generate parameters description from config file

* made pylint happy

* added checks description

* fixed links handling in desc and descl2 fields

* refine Core Parameters section

* removed excess quotes from default values

* fixed parameter type retrieve

* added note about auto-generated parameters

docs/Parameters.rst

@@ -1,3 +1,5 @@
+..  List of parameters is auto generated by LightGBM\helper\parameter_generator.py from LightGBM\include\LightGBM\config.h file.
+
 Parameters
 ==========
 
@@ -23,6 +25,8 @@ By using config files, one line can only contain one parameter. You can use ``#`
 
 If one parameter appears in both command line and config file, LightGBM will use the parameter in command line.
 
+.. start params list
+
 Core Parameters
 ---------------
 
@@ -721,6 +725,8 @@ This feature is only supported in command line version yet.
 
    -  output file name of converted model
 
+.. end params list
+
 Others
 ------
 

docs/README.rst

@@ -3,6 +3,10 @@ Documentation
 
 Documentation for LightGBM is generated using `Sphinx <http://www.sphinx-doc.org/>`__.
 
+List of parameters and their descriptions in `Parameters.rst <https://github.com/Microsoft/LightGBM/blob/master/docs/Parameters.rst>`__
+is generated automatically from comments in `config file <https://github.com/Microsoft/LightGBM/blob/master/include/LightGBM/config.h>`__
+by `this script <https://github.com/Microsoft/LightGBM/blob/master/helper/parameter_generator.py>`__.
+
 After each commit on ``master``, documentation is updated and published to `Read the Docs <https://lightgbm.readthedocs.io/>`__.
 
 Build

helper/parameter_generator.py

@@ -24,7 +24,7 @@ def GetParameterInfos(config_hpp):
             elif cur_key is not None:
                 line = line.strip()
                 if line.startswith("//"):
-                    tokens = line.split("//")[1].split("=")
+                    tokens = line[2:].split("=")
                     key = tokens[0].strip()
                     val = '='.join(tokens[1:]).strip()
                     if key not in cur_info:
@@ -54,7 +54,7 @@ def GetParameterInfos(config_hpp):
                             cur_info["name"] = [tokens[1][:-1].strip()]
                     member_infos[-1].append(cur_info)
                     cur_info = {}
-    return (keys, member_infos)
+    return keys, member_infos
 
 
 def GetNames(infos):
@@ -97,11 +97,72 @@ def SetOneVarFromString(name, type, checks):
     return ret
 
 
+def GenParameterDescription(sections, descriptions, params_rst):
+
+    def parse_check(check, reverse=False):
+        try:
+            idx = 1
+            float(check[idx:])
+        except ValueError:
+            idx = 2
+            float(check[idx:])
+        if reverse:
+            reversed_sign = {'<': '>', '>': '<', '<=': '>=', '>=': '<='}
+            return check[idx:], reversed_sign[check[:idx]]
+        else:
+            return check[idx:], check[:idx]
+
+    params_to_write = []
+    for section_name, section_params in zip(sections, descriptions):
+        params_to_write.append('{0}\n{1}'.format(section_name, '-' * len(section_name)))
+        for param_desc in section_params:
+            name = param_desc['name'][0]
+            default_raw = param_desc['default'][0]
+            default = default_raw.strip('"') if len(default_raw.strip('"')) > 0 else default_raw
+            param_type = param_desc.get('type', param_desc['inner_type'])[0].split(':')[-1].split('<')[-1].strip('>')
+            options = param_desc.get('options', [])
+            if len(options) > 0:
+                options_str = ', options: ``{0}``'.format('``, ``'.join([x.strip() for x in options[0].split(',')]))
+            else:
+                options_str = ''
+            aliases = param_desc.get('alias', [])
+            if len(aliases) > 0:
+                aliases_str = ', aliases: ``{0}``'.format('``, ``'.join([x.strip() for x in aliases[0].split(',')]))
+            else:
+                aliases_str = ''
+            checks = sorted(param_desc.get('check', []))
+            checks_len = len(checks)
+            if checks_len > 1:
+                number1, sign1 = parse_check(checks[0])
+                number2, sign2 = parse_check(checks[1], reverse=True)
+                checks_str = ', ``{0} {1} {2} {3} {4}``'.format(number2, sign2, name, sign1, number1)
+            elif checks_len == 1:
+                number, sign = parse_check(checks[0])
+                checks_str = ', ``{0} {1} {2}``'.format(name, sign, number)
+            else:
+                checks_str = ''
+            main_desc = '-  ``{0}``, default = ``{1}``, type = {2}{3}{4}{5}'.format(name, default, param_type, options_str, aliases_str, checks_str)
+            params_to_write.append(main_desc)
+            params_to_write.extend([' ' * 3 * int(desc[0][-1]) + '-  ' + desc[1] for desc in param_desc['desc']])
+
+    with open(params_rst) as original_params_file:
+        all_lines = original_params_file.read()
+        before, start_sep, _ = all_lines.partition('.. start params list\n\n')
+        _, end_sep, after = all_lines.partition('\n\n.. end params list')
+
+    with open(params_rst, "w") as new_params_file:
+        new_params_file.write(before)
+        new_params_file.write(start_sep)
+        new_params_file.write('\n\n'.join(params_to_write))
+        new_params_file.write(end_sep)
+        new_params_file.write(after)
+
+
 def GenParameterCode(config_hpp, config_out_cpp):
     keys, infos = GetParameterInfos(config_hpp)
     names = GetNames(infos)
     alias = GetAlias(infos)
-    str_to_write = "/// This file is auto generated by LightGBM\\helper\\parameter_generator.py\n"
+    str_to_write = "/// This file is auto generated by LightGBM\\helper\\parameter_generator.py from LightGBM\\include\\LightGBM\\config.h file.\n"
     str_to_write += "#include<LightGBM/config.h>\nnamespace LightGBM {\n"
     # alias table
     str_to_write += "std::unordered_map<std::string, std::string> Config::alias_table({\n"
@@ -151,8 +212,12 @@ def GenParameterCode(config_hpp, config_out_cpp):
     with open(config_out_cpp, "w") as config_out_cpp_file:
         config_out_cpp_file.write(str_to_write)
 
+    return keys, infos
+
 
 if __name__ == "__main__":
     config_hpp = os.path.join(os.path.pardir, 'include', 'LightGBM', 'config.h')
     config_out_cpp = os.path.join(os.path.pardir, 'src', 'io', 'config_auto.cpp')
-    GenParameterCode(config_hpp, config_out_cpp)
+    params_rst = os.path.join(os.path.pardir, 'docs', 'Parameters.rst')
+    sections, descriptions = GenParameterCode(config_hpp, config_out_cpp)
+    GenParameterDescription(sections, descriptions, params_rst)