Fix c++ module parsing and refactor (#8)

* Refactor modules

* Fix errors in modules

README.md

@@ -66,6 +66,7 @@ Reportedly working but no instructions yet.
 - DFHack's C++ functions that are exposed to Lue are **not included** (#4)
 - Initally opening DFHack can take a long time to load, especially on weaker hardware (#5)
 - LuaLS has a known problem with type-hinting inside of files that write to the `_ENV` global, if you are in a file writing to `_ENV` comment out those lines.
+- Types defined in C++ headers (like `NoblePosition`) are unsupported
 
 ## Credits
 

dist/library/modules/buildings.lua

@@ -9,7 +9,7 @@ dfhack.buildings = {}
 ---@return df.building
 function dfhack.buildings.findAtTile(pos) end
 
----@param pvec df.DFVector<building_civzonest>
+---@param pvec { [integer]: df.building_civzonest }
 ---@param pos df.coord
 ---@return boolean
 function dfhack.buildings.findCivzonesAt(pvec, pos) end
@@ -74,12 +74,12 @@ function dfhack.buildings.hasSupport(pos, size) end
 function dfhack.buildings.constructAbstract(bld) end
 
 ---@param bld df.building
----@param items df.DFVector<item>
+---@param items { [integer]: df.item }
 ---@return boolean
 function dfhack.buildings.constructWithItems(bld, items) end
 
 ---@param bld df.building
----@param items df.DFVector<job_item>
+---@param items { [integer]: df.job_item }
 ---@return boolean
 function dfhack.buildings.constructWithFilters(bld, items) end
 

dist/library/modules/burrows.lua

@@ -6,7 +6,7 @@
 ---@field setAssignedBlockTile function
 dfhack.burrows = {}
 
----@param pvec df.DFVector<map_block>
+---@param pvec { [integer]: df.map_block }
 ---@param burrow df.burrow
 ---@return nil
 function dfhack.burrows.listBlocks(pvec, burrow) end

dist/library/modules/filesystem.lua

@@ -5,17 +5,16 @@
 dfhack.filesystem = {}
 
 ---@param dir string
----@param files df.DFVector<std::string>
+---@param files { [integer]: string }
 ---@return integer
 function dfhack.filesystem.listdir(dir, files) end
 
 ---@param dir string
----@param std::map<std::string df.
----@param files df.bool>
----@param / df.int depth / = 10
----@param / df.bool include_prefix / = true
+---@param files { [string]: boolean }
+---@param depth integer
+---@param include_prefix boolean|nil
 ---@return integer
-function dfhack.filesystem.listdir_recursive(dir, std::map<std::string, files, /, /) end
+function dfhack.filesystem.listdir_recursive(dir, files, depth, include_prefix) end
 
 ---@return string
 function dfhack.filesystem.getcwd() end

dist/library/modules/gui.lua

@@ -50,7 +50,7 @@ function dfhack.gui.showAutoAnnouncement(type, pos, message, color, bright, unit
 ---@return boolean
 function dfhack.gui.autoDFAnnouncement(info, message) end
 
----@return df.Gui::DwarfmodeDims
+---@return unknown
 function dfhack.gui.getDwarfmodeViewDims() end
 
 ---@param x number
@@ -73,7 +73,7 @@ function dfhack.gui.revealInDwarfmodeMap(x, y, z, center, highlight) end
 function dfhack.gui.getMousePos(allow_out_of_bounds) end
 
 ---@param top df.viewscreen
----@return df.DFVector<std::string>
+---@return { [integer]: string }
 function dfhack.gui.getFocusStrings(top) end
 
 ---@param container df.widget_container

dist/library/modules/items.lua

@@ -22,11 +22,11 @@ function dfhack.items.getPosition(item) end
 function dfhack.items.getOuterContainerRef(spec_ref, item, init_ref) end
 
 ---@param item df.item
----@param items df.DFVector<item>
+---@param items { [integer]: df.item }
 ---@return nil
 function dfhack.items.getContainedItems(item, items) end
 
----@param mc df.MapExtras::MapCache
+---@param mc unknown
 ---@param item df.item
 ---@param building df.building_actual
 ---@param use_mode df.building_item_role_type

dist/library/modules/job.lua

@@ -6,7 +6,7 @@
 ---@field is_item_equal function
 dfhack.job = {}
 
----@param pvec df.DFVector<job>
+---@param pvec { [integer]: df.job }
 ---@param id_var integer
 ---@return boolean
 function dfhack.job.listNewlyCreated(pvec, id_var) end

dist/library/modules/screen.lua

@@ -13,30 +13,30 @@ function dfhack.screen.getMousePixels() end
 ---@return df.coord2d
 function dfhack.screen.getWindowSize() end
 
----@param pen df.Pen
+---@param pen unknown
 ---@param x integer
 ---@param y integer
 ---@param map boolean|nil
----@param df::graphic_viewportst::texpos_field df.int32_t
+---@param df::graphic_viewportst::texpos_field number
 ---@return boolean
 function dfhack.screen.paintTile(pen, x, y, map, df::graphic_viewportst::texpos_field) end
 
 ---@param x integer
 ---@param y integer
 ---@param map boolean|nil
----@param df::graphic_viewportst::texpos_field df.int32_t
----@return df.Pen
+---@param df::graphic_viewportst::texpos_field number
+---@return unknown
 function dfhack.screen.readTile(x, y, map, df::graphic_viewportst::texpos_field) end
 
----@param pen df.Pen
+---@param pen unknown
 ---@param x integer
 ---@param y integer
 ---@param text string
 ---@param map boolean|nil
 ---@return boolean
 function dfhack.screen.paintString(pen, x, y, text, map) end
 
----@param pen df.Pen
+---@param pen unknown
 ---@param x1 integer
 ---@param y1 integer
 ---@param x2 integer
@@ -57,7 +57,7 @@ function dfhack.screen.findGraphicsTile(pagename, x, y, ptile, pgs) end
 ---@return integer
 function dfhack.screen.keyToChar(key) end
 
----@param code df.char
+---@param code unknown
 ---@return df.interface_key
 function dfhack.screen.charToKey(code) end
 

dist/library/modules/units.lua

@@ -14,12 +14,12 @@ function dfhack.units.getPosition(unit) end
 ---@return nil
 function dfhack.units.getOuterContainerRef(spec_ref, unit, init_ref) end
 
----@param pvec df.DFVector<NoblePosition>
+---@param pvec { [integer]: unknown }
 ---@param unit df.unit
 ---@return boolean
 function dfhack.units.getNoblePositions(pvec, unit) end
 
----@param units df.DFVector<unit>
+---@param units { [integer]: df.unit }
 ---@param x1 number
 ---@param y1 number
 ---@param z1 number
@@ -29,13 +29,13 @@ function dfhack.units.getNoblePositions(pvec, unit) end
 ---@return boolean
 function dfhack.units.getUnitsInBox(units, x1, y1, z1, x2, y2, z2) end
 
----@param citizens df.vector<unit >
+---@param citizens { [integer]: df.unit }
 ---@param exclude_residents boolean|nil
 ---@param include_insane boolean|nil
 ---@return boolean
 function dfhack.units.getCitizens(citizens, exclude_residents, include_insane) end
 
----@param units df.vector<unit >
+---@param units { [integer]: df.unit }
 ---@param noble string
 ---@return boolean
 function dfhack.units.getUnitsByNobleRole(units, noble) end

lib/lua_ls.rb → lib/annotation.rb

@@ -2,7 +2,7 @@
 
 module DFHackLuaDefinitions
   # LuaLS annotation generator functions.
-  class LuaLS
+  module Annotation
     TYPES = %w[nil any boolean string number integer function table thread userdata lightuserdata].freeze
 
     # Keywords reserved by Lua that should not exist as identifiers.

lib/cpp.rb

@@ -1,55 +1,115 @@
 # frozen_string_literal: true
 
+require_relative 'annotation'
+
 module DFHackLuaDefinitions
-  class CPP
+  module CPP
+    # C++ types mapped to their (rough) Lua equivalents.
     TYPE_MAP = {
       'int' => 'integer',
       'double' => 'number',
       'float' => 'number',
       'bool' => 'boolean',
+      'string' => 'string',
       'std::string' => 'string',
-      'void' => 'nil',
-
-      # Structures
-      'int8_t' => 'number',
-      'uint8_t' => 'integer',
-      'int16_t' => 'number',
-      'uint16_t' => 'integer',
-      'int32_t' => 'number',
-      'uint32_t' => 'integer',
-      'int64_t' => 'number',
-      'uint64_t' => 'integer',
-      'size_t' => 'integer',
-      # 'enum-item' => 'integer',
-      # 'flag-bit' => 'integer',
-      'pointer' => 'integer',
-      # 'padding' => 'integer',
-      # 'stl-vector' => 'integer',
-      's-float' => 'number',
-      'd-float' => 'number',
-      'long' => 'number',
-      'ulong' => 'number',
-      'ptr-string' => 'DFPointer<string>',
-      'static-string' => 'string',
-      'stl-string' => 'string',
-      # 'stl-bit-vector' => 'boolean',
-      # 'df-flagarray' => 'boolean',
-      # 'stl-function' => 'function',
-      # TODO: Investigate a proper representation for these
-      'stl-mutex' => 'lightuserdata',
-      'stl-condition-variable' => 'lightuserdata',
-      'stl-deque' => 'lightuserdata',
-      'stl-fstream' => 'lightuserdata',
-      'stl-unordered-map' => 'lightuserdata',
-      'stl-future' => 'lightuserdata'
+      'void' => 'nil'
     }.freeze
 
     class << self
+      # WIP, volatile, here be Carps, you have been warned etc.
+      # Parses the `entry_point` c++ file for DFHack exposed Lua functions, then
+      # attempts to find the relevant module files and their function
+      # signatures.
+      def parse_cpp_modules(entry_point)
+        ignored_modules = %w[console]
+
+        file = File.read(entry_point)
+        directory = File.dirname(entry_point)
+
+        file.scan(/^static.*module\[\][\s\S]+?};/) do |cpp_module|
+          module_name = cpp_module[/\S+(?=_module\[\])/].gsub('dfhack_', '')
+
+          next if ignored_modules.include? module_name
+
+          module_file = if %w[dfhack internal].include? module_name
+                          file
+                        else
+                          File.read("#{directory}/modules/#{module_name.capitalize}.cpp")
+                        end
+
+          File.open("dist/library/modules/#{module_name}.lua", 'w') do |output|
+            print "Parsing: #{module_name}\n"
+            output << FILE_HEADER
+            output << "---@meta\n\n"
+
+            output << "---@class #{module_name}_module\n"
+
+            # Functions with signatures that are unlikely to be easily parsed.
+            cpp_module.scan(/(?:WRAP_VERSION_FUNC|WRAPN)\(([^)]+)\)/) do |match|
+              function_name = match[0].split(', ')[0]
+              output << "---@field #{function_name} function\n"
+            end
+
+            prefix = module_name == 'dfhack' ? '' : 'dfhack.'
+            namespace = module_name == 'dfhack' ? '' : "#{module_name.capitalize}::"
+            output << "#{prefix}#{module_name} = {}\n\n"
+
+            functions = []
+
+            # Guessing here a little bit.
+            file.scan(/^static.*#{module_name}_funcs\[\][\s\S]+?};/) do |funcs|
+              funcs.scan(/{([^}\n]+)}/) do
+                match = Regexp.last_match(1)
+                next if match =~ /NULL/
+
+                function_name = match.split(',')[0].strip.gsub(/"/, '')
+                signature_name = match.split(',')[1].strip.gsub('"', '').gsub("#{module_name}_", '')
+
+                module_file[/^(?:static\s)?(?:DFHACK_EXPORT\s)?(\S+).*?#{namespace}#{signature_name}\s?\(([^)]+)?\)/]
+                next unless Regexp.last_match
+
+                functions << DFHackLuaDefinitions::CPP.parse_function(Regexp.last_match, module_name:, prefix:,
+                                                                                         function_name:)
+              end
+            end
+
+            cpp_module.scan(/(?:WRAP|WRAPM)\((.+)?\),?/) do |function_name,|
+              function_name = Regexp.last_match(1) if function_name =~ /,\s?(\S+)/
+              signature = "#{namespace}#{function_name}"
+
+              module_file[/^(?:static\s)?(?:DFHACK_EXPORT\s)?(\S+).*?#{signature}\s?\(([^)]+)?\)/]
+
+              next unless Regexp.last_match
+
+              functions << DFHackLuaDefinitions::CPP.parse_function(Regexp.last_match, module_name:, prefix:,
+                                                                                       function_name:)
+            end
+
+            output << functions.join
+          end
+        end
+      end
+
       def parse_type(string)
+        # We'll clean this later.
+        string = string.strip
+
         return TYPE_MAP[string] if TYPE_MAP[string]
+        return XML::TYPE_MAP[string] if XML::TYPE_MAP[string]
+
+        string = string.gsub(/df::/, 'df.') if string
 
-        string = string.gsub(/df::/, '') if string
-        string = string.gsub(/std::vector/, 'DFVector') if string
+        # TODO: Move this out to annotations and give it more glue.
+        return "{ [integer]: #{parse_type(Regexp.last_match(1))} }" if string[/(?:std::)?vector<([^>]+)>/]
+
+        if string[/(?:std::)?map<([^>]+)>/]
+          key = parse_type(Regexp.last_match(1).split(',')[0])
+          value = parse_type(Regexp.last_match(1).split(',')[1])
+          return "{ [#{key}]: #{value} }"
+        end
+
+        # TODO: Temporary, we'll do some funky module resolution later.
+        return 'unknown' unless string.include? 'df.'
 
         string
       end
@@ -69,15 +129,16 @@ def parse_function(match, module_name:, prefix:, function_name:)
 
         if captures[1]
           # TODO: Naming convention or actual compiler behaviour?
-          arguments = captures[1].split(',').reject.with_index { |arg, index| arg[/(&\s*out)|lua_State/] && index.zero? }
-          arguments = arguments.map { |arg| arg.gsub(/const\s+|[*&]/, '').strip }
+          arguments = captures[1].split(/,(?![^<>]*>)/).reject.with_index do |arg, index|
+            arg[/(&\s*out)|lua_State/] && index.zero?
+          end
+          arguments = arguments.map { |arg| arg.gsub(%r{/\*[^/]+/}, '').gsub(/const\s+|[*&]/, '').strip }
           arguments = arguments&.map do |argument|
             type, _, name = argument.rpartition(' ')
             type = DFHackLuaDefinitions::CPP.parse_type(type)
-            type = "df.#{type}" unless DFHackLuaDefinitions::LuaLS::TYPES.include? type
 
             {
-              name: DFHackLuaDefinitions::LuaLS.safe_name(DFHackLuaDefinitions::CPP.sanitize(name)),
+              name: Annotation.safe_name(DFHackLuaDefinitions::CPP.sanitize(name)),
               type: type == 'boolean' ? 'boolean|nil' : type.strip
             }
           end&.compact
@@ -89,9 +150,6 @@ def parse_function(match, module_name:, prefix:, function_name:)
         end
 
         if return_type
-          # Namespacing
-          return_type = "df.#{return_type}" unless DFHackLuaDefinitions::LuaLS::TYPES.include? return_type
-
           annotation << "---@return #{return_type.gsub(/const\s+|[*&]/, '')}\n" if return_type
         else
           annotation << "---@return unknown\n"