Quality of life tweaks

.dockerignore

@@ -0,0 +1,2 @@
+*.gem
+pkg

Dockerfile

@@ -0,0 +1,39 @@
+# Dependencies we need available when running
+ARG RUN_DEPS="ruby-dev llvm libclang-3.8-dev python-pygments libffi6"
+
+FROM debian:stretch-slim as builder
+
+# Extra dependencies we need in the build container
+ARG BUILD_DEPS="git cmake pkg-config build-essential libffi-dev libssl-dev"
+
+# takes the global value defined above
+ARG RUN_DEPS
+
+# We need these packages to build ruby extensions, rugged and to parse the C
+# code. pygments is there to highlight the code examples.
+RUN apt update && \
+    apt install -y --no-install-recommends ${BUILD_DEPS} ${RUN_DEPS} && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /docurium
+
+# This is here so we can provide a unique argument per run so docker does not
+# consider the rest of the file cached and wealways install the latest version
+# of docurium
+ARG CACHEBUST=1
+
+COPY . /docurium/
+RUN gem build docurium && gem install docurium-*.gem --no-ri --no-rdoc
+
+FROM debian:stretch-slim
+
+# takes the global value defined above
+ARG RUN_DEPS
+RUN apt update && \
+    apt install -y --no-install-recommends ${RUN_DEPS} && \
+    apt clean && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY --from=builder /var/lib/gems/ /var/lib/gems/
+COPY --from=builder /usr/local/bin/cm /usr/local/bin/cm

Rakefile

@@ -5,10 +5,22 @@ require 'rubygems/package_task'
 task :default => :test
 
 gemspec = Gem::Specification::load(File.expand_path('../docurium.gemspec', __FILE__))
-Gem::PackageTask.new(gemspec) do |pkg|
-end
+Gem::PackageTask.new(gemspec).define
 
 Rake::TestTask.new do |t|
   t.libs << 'libs' << 'test'
   t.pattern = 'test/**/*_test.rb'
 end
+
+namespace :docker do
+  task :build do
+    sh("docker build . -t docurium")
+  end
+
+  task :run, [:on] do |t, args|
+    src = File.realpath(args[:on])
+    sh("echo \"# cd /workspace && cm doc api.docurium\"")
+    sh("docker run -v #{src}:/workspace -it docurium")
+  end
+end
+

docurium.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |s|
   s.add_dependency "rugged", "~>0.21"
   s.add_dependency "redcarpet", "~>3.0"
   s.add_dependency "ffi-clang", "~> 0.5"
+  s.add_dependency "parallel", "~> 1.17.0"
   s.add_development_dependency "rake", "~> 12"
   s.add_development_dependency "minitest", "~> 5.11"
 

lib/docurium.rb

@@ -11,6 +11,7 @@
 require 'rugged'
 require 'redcarpet'
 require 'redcarpet/compat'
+require 'parallel'
 require 'thread'
 
 # Markdown expects the old redcarpet compat API, so let's tell it what
@@ -118,69 +119,41 @@ def format_examples!(data, version)
   def generate_doc_for(version)
     index = Rugged::Index.new
     read_subtree(index, version, option_version(version, 'input', ''))
+
     data = parse_headers(index, version)
-    data
+    examples = format_examples!(data, version)
+    [data, examples]
   end
 
   def process_project(versions)
-    nversions = versions.size
-    output = Queue.new
-    pipes = {}
-    versions.each do |version|
-      # We don't need to worry about joining since this process is
-      # going to die immediately
-      read, write = IO.pipe
-      pid = Process.fork do
-        read.close
-
-        data = generate_doc_for(version)
-        examples = format_examples!(data, version)
-
-        Marshal.dump([version, data, examples], write)
-        write.close
-      end
-
-      pipes[pid] = read
-      write.close
-    end
-
-    print "Generating documentation [0/#{nversions}]\r"
-
-    # This may seem odd, but we need to keep reading from the pipe or
-    # the buffer will fill and they'll block and never exit. Therefore
-    # we can't rely on Process.wait to tell us when the work is
-    # done. Instead read from all the pipes concurrently and send the
-    # ruby objects through the queue.
-    Thread.abort_on_exception = true
-    pipes.each do |pid, read|
-      Thread.new do
-        result = read.read
-        output << Marshal.load(result)
-      end
-    end
-
-    for i in 1..nversions
-      version, data, examples = output.pop
-
+    nversions = versions.count
+    Parallel.each_with_index(versions, finish: -> (version, index, result) do
+      data, examples = result
       # There's still some work we need to do serially
       tally_sigs!(version, data)
       force_utf8(data)
 
+      puts "Adding documentation for #{version} [#{index}/#{nversions}]"
+
       # Store it so we can show it at the end
       @head_data = data if version == 'HEAD'
 
-      yield i, nversions, version, data, examples if block_given?
+      yield index, version, result if block_given?
+
+    end) do |version, index|
+      puts "Generating documentation for #{version} [#{index}/#{nversions}]"
+      generate_doc_for(version)
     end
   end
 
-  def generate_docs(options)
+  def generate_docs
     output_index = Rugged::Index.new
     write_site(output_index)
     @tf = File.expand_path(File.join(File.dirname(__FILE__), 'docurium', 'layout.mustache'))
     versions = get_versions
     versions << 'HEAD'
     # If the user specified versions, validate them and overwrite
-    if !(vers = (options[:for] || [])).empty?
+    if !(vers = (@cli_options[:for] || [])).empty?
       vers.each do |v|
         next if versions.include?(v)
         puts "Unknown version #{v}"
@@ -189,8 +162,9 @@ def generate_docs(options)
       versions = vers
     end
 
-    process_project(versions) do |i, version, data, examples|
-      @repo.write(data.to_json, :blob)
+    process_project(versions) do |i, version, result|
+      data, examples = result
+      sha = @repo.write(data.to_json, :blob)
 
       print "Generating documentation [#{i}/#{versions.count}]\r"
 
@@ -314,7 +288,7 @@ def initialize(warning, type, identifier, opts = {})
 
     def message
       msg = self._message
-      msg.shift % msg.map {|a| self.send(a).to_s } if msg.kind_of?(Array)
+      msg.kind_of?(Array) ? msg.shift % msg.map {|a| self.send(a).to_s } : msg
     end
   end
 
@@ -457,7 +431,7 @@ def find_subtree(version, path)
 
   def read_subtree(index, version, path)
     tree = find_subtree(version, path)
-    index.read_tree(tree)
+    index.read_tree(tree) rescue nil
   end
 
   def valid_config(file)

lib/docurium/docparser.rb

@@ -70,7 +70,15 @@ def parse_file(orig_filename, opts = {})
       args = includes.map { |path| "-I#{path}" }
       args << '-ferror-limit=1'
 
-      tu = Index.new(true, true).parse_translation_unit(filename, args, @unsaved, {:detailed_preprocessing_record => 1})
+      attempts = 3
+      begin
+        tu = Index.new(true, true).parse_translation_unit(filename, args, @unsaved, {:detailed_preprocessing_record => 1})
+      rescue Exception => e
+        puts "Exception raised while parsing #{filename}:#{e}"
+        puts e.inspect
+        retry if attempts > 0
+        return []
+      end
 
       recs = []
 
@@ -87,7 +95,6 @@ def parse_file(orig_filename, opts = {})
         #  :continue
         #end
 
-        next :continue if cursor.comment.kind == :comment_null
         next :continue if cursor.spelling == ""
 
         extent = cursor.extent
@@ -111,6 +118,14 @@ def parse_file(orig_filename, opts = {})
         when :cursor_typedef_decl
           debug "have typedef #{cursor.spelling} #{cursor.underlying_type.spelling}"
           rec.update extract_typedef(cursor)
+        when :cursor_variable
+          next :continue
+        when :cursor_macro_definition
+          next :continue
+        when :cursor_inclusion_directive
+          next :continue
+        when :cursor_macro_expansion
+          next :continue
         else
           raise "No idea how to deal with #{cursor.kind}"
         end

test/docurium_test.rb

@@ -28,10 +28,17 @@ def teardown
     FileUtils.remove_entry(@dir)
   end
 
+  def test_can_parse
+    refute_nil @data
+    assert_equal [:files, :functions, :callbacks, :globals, :types, :prefix, :groups], @data.keys
+    files = %w(blob.h callback.h cherrypick.h commit.h common.h errors.h index.h object.h odb.h odb_backend.h oid.h refs.h repository.h revwalk.h signature.h tag.h tree.h types.h)
+    assert_equal files, @data[:files].map {|d| d[:file] }
+  end
+
   def test_can_parse_headers
     keys = @data.keys.map { |k| k.to_s }.sort
     assert_equal ['callbacks', 'files', 'functions', 'globals', 'groups', 'prefix', 'types'], keys
-    assert_equal 155, @data[:functions].size
+    assert_equal 159, @data[:functions].size
   end
 
   def test_can_extract_enum_from_define
@@ -164,15 +171,49 @@ def test_can_parse_function_cast_args
     assert_equal expect_comment.split("\n"), func[:comments].split("\n")
   end
 
+  def test_can_handle_bulleted_lists
+    type = @data[:types].find {|name, data| name == 'git_repository_init_options' }
+    refute_nil type
+    expect_comment = <<-EOF
+<p>This contains extra options for <code>git_repository_init_ext</code> that enable
+ additional initialization features.  The fields are:</p>
+
+<ul>
+<li>flags - Combination of GIT_REPOSITORY_INIT flags above.</li>
+<li>mode  - Set to one of the standard GIT_REPOSITORY_INIT_SHARED_...
+    constants above, or to a custom value that you would like.</li>
+<li>workdir_path - The path to the working dir or NULL for default (i.e.
+    repo_path parent on non-bare repos).  IF THIS IS RELATIVE PATH,
+    IT WILL BE EVALUATED RELATIVE TO THE REPO_PATH.  If this is not
+    the &quot;natural&quot; working directory, a .git gitlink file will be
+    created here linking to the repo_path.</li>
+<li>description - If set, this will be used to initialize the &quot;description&quot;
+    file in the repository, instead of using the template content.</li>
+<li>template_path - When GIT_REPOSITORY_INIT_EXTERNAL_TEMPLATE is set,
+    this contains the path to use for the template directory.  If
+    this is NULL, the config or default directory options will be
+    used instead.</li>
+<li>initial_head - The name of the head to point HEAD at.  If NULL, then
+    this will be treated as &quot;master&quot; and the HEAD ref will be set
+    to &quot;refs/heads/master&quot;.  If this begins with &quot;refs/&quot; it will be
+    used verbatim; otherwise &quot;refs/heads/&quot; will be prefixed.</li>
+<li>origin_url - If this is non-NULL, then after the rest of the
+    repository initialization is completed, an &quot;origin&quot; remote
+    will be added pointing to this URL.</li>
+</ul>
+EOF
+    assert_equal expect_comment, type[1][:comments]
+  end
+
   def test_can_get_the_full_description_from_multi_liners
     func = @data[:functions]['git_commit_create_o']
     desc = "<p>Create a new commit in the repository using <code>git_object</code>\n instances as parameters.</p>\n"
     assert_equal desc, func[:description]
   end
 
   def test_can_group_functions
-    groups = %w(blob cherrypick commit index lasterror object odb oid reference repository revwalk signature strerror tag tree treebuilder work)
-    assert_equal groups, @data[:groups].map {|g| g[0]}.sort
+    groups = %w(blob cherrypick commit index lasterror object odb oid reference repository revwalk signature strarray strerror tag tree treebuilder work)
+    assert_equal groups, @data[:groups].map {|g| g[0]}
     group, funcs = @data[:groups].first
     assert_equal 'blob', group
     assert_equal 6, funcs.size

test/fixtures/git2/types.h

@@ -50,6 +50,7 @@ GIT_BEGIN_DECL
  * stat() functions, for all platforms.
  */
 #include <sys/types.h>
+#include <stdint.h>
 
 #if defined(_MSC_VER)