Enable Intersphinx support - Take 2 (#88)

* Enables inter-sphinx based cross-references.

* Removes unused Jekyll plugins.

* Brings in Sphinx usage improvements from distro versioning work.

* Fixes bad title in documentation root pages.

.gitignore

@@ -3,6 +3,7 @@ _site
 _devel
 _roswiki
 _cache
-_sphinx
 _remotes
 _plugins_data
+_sphinx/_build
+_sphinx/repos

Gemfile

@@ -5,7 +5,6 @@ gem 'colorator'
 #gem 'git'
 gem 'github-markup'
 gem 'jekyll'
-gem 'jekyll-relative-links'
 gem 'jekyll-sitemap'
 gem 'liquid'#, '~> 2.6.2'
 gem 'mercurial-ruby'

README.md

@@ -137,7 +137,7 @@ Additionally, some make targets are provided for convenience:
 - To skip everything but repo discovering:
 
   ```bash
-  make discover   #
+  make discover
   ```
 
 - To skip everything but repo updates:

_config.yml

@@ -101,9 +101,5 @@ repos_per_page: 50
 # uncomment the following line for testing (limit the number of indexed repos)
 max_repos: 00
 
-readme_index:
-  remove_originals: true
-
 plugins:
   - jekyll-sitemap
-  - jekyll-relative-links

_layouts/doc.html

@@ -6,34 +6,19 @@
   <div class="container-fluid">
     <div class="row">
       <ol class="breadcrumb">
-        <li><a href="{{site.baseurl}}/">Home</a></li>
-        {% assign splitted_url = page.url | split:"/" %}
-        {% assign repo_name = splitted_url[2] %}
-        {% assign page_name = splitted_url[-1] %}
-
-        {% assign description = site.docs_repos[repo_name].description %}
-
-        {% if repo_name != page_name %}
-          <li><a href="{{site.baseurl}}/doc/{{repo_name}}/">{{ description }}</a></li>
-          {% if page.title == "" or page.title == nil  %}
-            <li class="active">{{ page_name | remove: ".html" }}</li>
-          {% else %}
-            <li class="active">{{ page.title }}</li>
-          {% endif %}
-        {% else %}
-          <li class="active">{{ description }}</li>
-        {% endif %}
+        <li><a href="{{ site.baseurl }}/">Home</a></li>
+        <li><a href="{{ page.docs_baseurl }}/">{{ page.docs_title }}</a></li>
+        {% unless page.at_root %}
+        <li class="active">{{ page.title }}</li>
+        {% endunless %}
       </ol>
-      {% if page.edit_url %}
+    </div>
+    <div class="row">
       <a type="button" href="{{ page.edit_url }}" class="btn btn-success pull-right">
         Edit on Github
       </a>
-      {% endif %}
-    </div>
-    <div class="row">
-      <div class="col-md-10">  
-        {{ content }}
-      </div>
+      {{ content }}
+      &nbsp;
     </div>
   </div>
 </div>

_plugins/docs_generator.rb

@@ -1,3 +1,4 @@
+require 'addressable'
 require 'fileutils'
 require 'nokogiri'
 
@@ -17,94 +18,91 @@ def initialize(config = {})
   end
 
   def generate(site)
-    doc_repos = site.config['docs_repos']
-    docfiles = search_docfiles(doc_repos)
-    copy_docs_to_sphinx_location(docfiles)
-    docs_data = convert_with_sphinx(docfiles)
-    docs_data.each do |repo_name, repo_docs|
-      repo_docs.each do |key, doc_data|
-        site.pages << DocPage.new(site, repo_name, doc_data)
+    all_repos = site.data['remotes']['repositories']
+    site.config['docs_repos'].each do |repo_name, repo_options|
+      next unless all_repos.key? repo_name
+      repo_data = all_repos[repo_name]
+      repo_description = repo_options['description'] || repo_name
+      convert_with_sphinx(repo_name, repo_data).each do |path, content|
+        site.pages << DocPage.new(
+          site, repo_name, repo_description, path, content
+        )
       end
     end
   end
 
-  def copy_docs_to_sphinx_location(docfiles)
-    docfiles.each do |repo_name, files|
-      origin_dir = File.join("_remotes", repo_name)
-      destination_dir = File.join("_sphinx", "repos", repo_name)
-      unless File.directory?(destination_dir)
-        FileUtils.makedirs(destination_dir)
-      end
-      files.each do |file|
-        if file.scan(/readme/i).length > 0
-          dest_name = "index" + File.extname(file)
-        else
-          dest_name = File.basename(file)
-        end
-        FileUtils.copy_entry(file, File.join(destination_dir, dest_name), preserve = true)
+  def copy_docs(src_path, dst_path)
+    copied_docs = Hash.new
+    src_path = Pathname.new(src_path)
+    Dir.glob(File.join(src_path, '**/*.rst'),
+             File::FNM_CASEFOLD).each do |src_doc_path|
+      src_doc_path = Pathname.new(src_doc_path)
+      dst_doc_path = Pathname.new(File.join(
+        dst_path, src_doc_path.relative_path_from(src_path)
+      ).sub(/readme\.rst$/i, 'index.rst'))
+      unless File.directory? File.dirname(dst_doc_path)
+        FileUtils.makedirs(File.dirname(dst_doc_path))
       end
+      FileUtils.copy_entry(src_doc_path, dst_doc_path, preserve = true)
+      copied_docs[src_doc_path] = dst_doc_path
     end
+    copied_docs
   end
 
-  def search_docfiles(doc_repos_hash)
-    docfiles_hash = Hash.new
-    doc_repos_hash.each do |repo_name, repo|
-      docs_in_repo = Dir.glob(File.join("_remotes", repo_name) + '**/*.{md, rst}', File::FNM_CASEFOLD)
-      docfiles_hash[repo_name] = docs_in_repo
+  def generate_edit_url(repo_data, original_filepath)
+    is_https = repo_data['url'].include? "https"
+    is_github = repo_data['url'].include? "github.com"
+    is_bitbucket = repo_data['url'].include? "bitbucket.org"
+    unless is_github or is_bitbucket
+      raise ValueError("Cannot generate edition URL. Unknown organization for repository: #{repo_data['url']}")
+    end
+    if is_https
+      uri = URI(repo_data['url'])
+      host = uri.host
+      organization, repo = uri.path.split("/").reject { |c| c.empty? }
+    else # ssh
+      host, path = repo_data['url'].split("@")[1].split(":")
+      organization, repo = path.split("/")
+    end
+    repo.chomp!(".git") if repo.end_with? ".git"
+    if is_github
+      edit_url = "https://#{host}/#{organization}/#{repo}/edit/#{repo_data['version']}"
+      return File.join(edit_url, original_filepath)
+    elsif is_bitbucket
+      edit_url = "https://#{host}/#{organization}/#{repo}/src/#{repo_data['version']}"
+      return File.join(edit_url, original_filepath) +
+             "?mode=edit&spa=0&at=#{repo_data['version']}&fileviewer=file-view-default"
     end
-    docfiles_hash
   end
 
-  def convert_with_sphinx(docfiles)
-    json_files_data = Hash.recursive
-    docfiles.each do |repo_name, files|
-      repo_path = File.join('_sphinx', 'repos', repo_name)
-      command = "sphinx-build -b json -c _sphinx #{repo_path} _sphinx/_build/#{repo_name}"
-      pid = Kernel.spawn(command)
-      Process.wait pid
-
-      files.each do |file|
-        leading_single_quotation = /^'/
-        trailing_single_quotation = /'$/
-        file.sub!(/#{leading_single_quotation} | #{trailing_single_quotation}/, '')
-        name = File.basename(file, File.extname(file))
-        if name.scan(/readme/i).length > 0
-          name = "index"
-        end
-        json_file = File.join('_sphinx/_build/', repo_name, name + '.fjson')
-        next unless File.file?(json_file)
-        json_content = File.read(json_file)
-        json_files_data[repo_name][name] = JSON.parse(json_content)
-        json_files_data[repo_name][name]["relative_path"] = file
-
-        # Generate HTML tables that were left with markdown syntax by sphinx.
-        # Please refer to this issue for more information:
-        # https://github.com/rtfd/recommonmark/issues/3
-        # Although there's a sphinx-markdown-tables extension now, its
-        # functionality is limited and doesn't fit our requirements.
-        html_doc = Nokogiri::HTML(JSON.parse(json_content)["body"])
-        html_doc.css('p').each do |paragraph|
-          if paragraph.content.strip[0] == "|"
-            html_table = Kramdown::Document.new(paragraph.inner_html).to_html
-            nokogiri_table = Nokogiri::HTML(html_table)
-            next unless nokogiri_table.css('td').length > 0
-            nokogiri_table.css('table').each do |table|
-              # Fixes table by removing the second row that's always filled
-              # with hyphens, as a consequence of conversion.
-              table.css('tr').each_with_index do |tr, index|
-                if index == 1
-                  tr.remove
-                  break
-                end
-              end
-              table.set_attribute("class", "table table-striped table-dark")
-            end
-            paragraph.replace(nokogiri_table.to_html)
-          end
-        end
-        json_files_data[repo_name][name]["body"] = html_doc.css('body').first.inner_html 
+  def convert_with_sphinx(repo_name, repo_data)
+    repo_path = Pathname.new(File.join("_remotes", repo_name))
+    in_path = Pathname.new(File.join('_sphinx', 'repos', repo_name))
+    FileUtils.rm_r(in_path) if File.directory? in_path
+    copied_docs_paths = copy_docs(repo_path, in_path)
+    return if copied_docs_paths.empty?
+    out_path = Pathname.new(File.join('_sphinx', '_build', repo_name))
+    FileUtils.rm_r(out_path) if File.directory? out_path
+    FileUtils.makedirs(out_path)
+    command = "sphinx-build -b json -c _sphinx #{in_path} #{out_path}"
+    pid = Kernel.spawn(command)
+    Process.wait pid
+    repo_content = Hash.recursive
+    copied_docs_paths.each do |src_path, dst_path|
+      json_path = File.join(
+        out_path, dst_path.relative_path_from(in_path)
+      ).sub(File.extname(dst_path), '.fjson')
+      next unless File.file? json_path
+      json_content = JSON.parse(File.read(json_path))
+      json_content["edit_url"] = generate_edit_url(
+        repo_data, src_path.relative_path_from(repo_path)
+      )
+      permalink = json_content["current_page_name"]
+      if File.basename(permalink) == "index"
+        permalink = File.dirname(permalink)
       end
+      repo_content[permalink] = json_content
     end
-    json_files_data
+    repo_content
   end
 end

_ruby_libs/pages.rb

@@ -172,23 +172,22 @@ def initialize(site, sort_id, n_list_pages, page_index, list, default=false)
 end
 
 class DocPage < Jekyll::Page
-  def initialize(site, repo_name, page_data)
-    basepath = File.join('doc', repo_name)
+  def initialize(site, docs_name, docs_title, page_name, page_data)
     @site = site
     @base = site.source
+    @dir  = "doc/#{docs_name}/#{page_name}"
     @name = "index.html"
-    if page_data["current_page_name"].scan(/index|readme/i).length > 0
-      @dir = "doc/#{repo_name}/"
-    else
-      @dir = "doc/#{repo_name}/#{page_data["current_page_name"]}/"
-    end
     self.process(@name)
     self.data ||= {}
+    self.content = page_data['body']
     self.data['layout'] = "doc"
-    self.content = page_data["body"]
-    self.data['file_extension'] = page_data["page_source_suffix"]
-    self.data['file_relpath'] = page_data["relative_path"]
-    self.data['title'] = page_data["current_page_name"]
+    self.data['at_root'] = (page_name == '.')
+    self.data['title'] = File.basename(
+      page_name != '.' ? page_name.gsub('-', ' ') : docs_title
+    )
+    self.data['edit_url'] = page_data['edit_url']
+    self.data['docs_baseurl'] = "#{site.baseurl}/doc/#{docs_name}"
+    self.data['docs_title'] = docs_title
   end
 end