From f2f084db8051308f5d156240fa3d628f195b8927 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sun, 16 Jun 2019 15:28:42 +0200 Subject: [PATCH 1/3] Delegate our parallel processing to a gem --- docurium.gemspec | 1 + lib/docurium.rb | 63 ++++++++++++------------------------------------ 2 files changed, 16 insertions(+), 48 deletions(-) diff --git a/docurium.gemspec b/docurium.gemspec index ca091c7dc..ce9597aec 100644 --- a/docurium.gemspec +++ b/docurium.gemspec @@ -20,6 +20,7 @@ Gem::Specification.new do |s| s.add_dependency "rugged", "~>0.21" s.add_dependency "redcarpet", "~>3.0" s.add_dependency "ffi-clang", "~> 0.5" + s.add_dependency "parallel", "~> 1.17.0" s.add_development_dependency "rake", "~> 12" s.add_development_dependency "minitest", "~> 5.11" diff --git a/lib/docurium.rb b/lib/docurium.rb index 0974e0e32..7a2e27663 100644 --- a/lib/docurium.rb +++ b/lib/docurium.rb @@ -10,6 +10,7 @@ require 'rugged' require 'redcarpet' require 'redcarpet/compat' +require 'parallel' require 'thread' # Markdown expects the old redcarpet compat API, so let's tell it what @@ -135,68 +136,34 @@ def generate_docs(options) versions = vers end - nversions = versions.size - output = Queue.new - pipes = {} - versions.each do |version| - # We don't need to worry about joining since this process is - # going to die immediately - read, write = IO.pipe - pid = Process.fork do - read.close - - data = generate_doc_for(version) - examples = format_examples!(data, version) - - Marshal.dump([version, data, examples], write) - write.close - end - - pipes[pid] = read - write.close - end - - print "Generating documentation [0/#{nversions}]\r" head_data = nil - - # This may seem odd, but we need to keep reading from the pipe or - # the buffer will fill and they'll block and never exit. Therefore - # we can't rely on Process.wait to tell us when the work is - # done. Instead read from all the pipes concurrently and send the - # ruby objects through the queue. - Thread.abort_on_exception = true - pipes.each do |pid, read| - Thread.new do - result = read.read - output << Marshal.load(result) - end - end - - for i in 1..nversions - version, data, examples = output.pop - + nversions = versions.count + Parallel.each_with_index(versions, finish: -> (version, index, result) do + version, data, examples = result # There's still some work we need to do serially tally_sigs!(version, data) force_utf8(data) sha = @repo.write(data.to_json, :blob) - print "Generating documentation [#{i}/#{nversions}]\r" + puts "Adding documentation for #{version} [#{index}/#{nversions}]" # Store it so we can show it at the end - if version == 'HEAD' - head_data = data - end + head_data = data if version == 'HEAD' output_index.add(:path => "#{version}.json", :oid => sha, :mode => 0100644) examples.each do |path, id| output_index.add(:path => path, :oid => id, :mode => 0100644) end + end) do |version, index| + puts "Generating documentation for #{version} [#{index}/#{nversions}]" + data = generate_doc_for(version) + examples = format_examples!(data, version) + [version, data, examples] + end - if head_data - puts '' - show_warnings(data) - end - + if head_data + puts '' + show_warnings(head_data) end # We tally the signatures in the order they finished, which is From 38564854b782aee123b1ccdbd452ee9415b73f06 Mon Sep 17 00:00:00 2001 From: Etienne Samson Date: Sat, 30 Nov 2019 22:16:09 +0100 Subject: [PATCH 2/3] Make doc generation method standalone --- lib/docurium.rb | 52 +++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/lib/docurium.rb b/lib/docurium.rb index 7a2e27663..cf8658a34 100644 --- a/lib/docurium.rb +++ b/lib/docurium.rb @@ -18,12 +18,13 @@ Rocco::Markdown = RedcarpetCompat class Docurium - attr_accessor :branch, :output_dir, :data + attr_accessor :branch, :output_dir, :data, :head_data def initialize(config_file, repo = nil) raise "You need to specify a config file" if !config_file raise "You need to specify a valid config file" if !valid_config(config_file) @sigs = {} + @head_data = nil @repo = repo || Rugged::Repository.discover(config_file) end @@ -116,8 +117,31 @@ def format_examples!(data, version) def generate_doc_for(version) index = Rugged::Index.new read_subtree(index, version, option_version(version, 'input', '')) - data = parse_headers(index, version) - data + + data = parse_headers(index, version, reference) + examples = format_examples!(data, version) + [data, examples] + end + + def process_project(versions) + nversions = versions.count + Parallel.each_with_index(versions, finish: -> (version, index, result) do + data, examples = result + # There's still some work we need to do serially + tally_sigs!(version, data) + force_utf8(data) + + puts "Adding documentation for #{version} [#{index}/#{nversions}]" + + # Store it so we can show it at the end + @head_data = data if version == 'HEAD' + + yield index, version, result if block_given? + + end) do |version, index| + puts "Generating documentation for #{version} [#{index}/#{nversions}]" + generate_doc_for(version) + end end def generate_docs(options) @@ -136,29 +160,15 @@ def generate_docs(options) versions = vers end - head_data = nil - nversions = versions.count - Parallel.each_with_index(versions, finish: -> (version, index, result) do - version, data, examples = result - # There's still some work we need to do serially - tally_sigs!(version, data) - force_utf8(data) - sha = @repo.write(data.to_json, :blob) - - puts "Adding documentation for #{version} [#{index}/#{nversions}]" - - # Store it so we can show it at the end - head_data = data if version == 'HEAD' + process_project(versions) do |i, version, result| + print "Writing documentation [#{i}/#{versions.count}]\r" + data, examples = result + sha = @repo.write(data.to_json, :blob) output_index.add(:path => "#{version}.json", :oid => sha, :mode => 0100644) examples.each do |path, id| output_index.add(:path => path, :oid => id, :mode => 0100644) end - end) do |version, index| - puts "Generating documentation for #{version} [#{index}/#{nversions}]" - data = generate_doc_for(version) - examples = format_examples!(data, version) - [version, data, examples] end if head_data From ebc7ce2e132e455ff843555992f2069e887a8cb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20Mart=C3=ADn=20Nieto?= Date: Thu, 19 Dec 2019 14:06:03 +0000 Subject: [PATCH 3/3] Return the original message in warnings if they're just a string Due to the `if` we'd return `nil` if the message was not an array. --- lib/docurium.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/docurium.rb b/lib/docurium.rb index 111c7fdf4..0bf86a2a3 100644 --- a/lib/docurium.rb +++ b/lib/docurium.rb @@ -288,7 +288,7 @@ def initialize(warning, type, identifier, opts = {}) def message msg = self._message - msg.shift % msg.map {|a| self.send(a).to_s } if msg.kind_of?(Array) + msg.kind_of?(Array) ? msg.shift % msg.map {|a| self.send(a).to_s } : msg end end