diff --git a/.gitignore b/.gitignore index 1b2a37a..3e7f0bd 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ _site _devel _roswiki _cache -_sphinx _remotes _plugins_data +_sphinx/_build +_sphinx/repos \ No newline at end of file diff --git a/Gemfile b/Gemfile index 81ce7a9..8c02d0c 100644 --- a/Gemfile +++ b/Gemfile @@ -5,7 +5,6 @@ gem 'colorator' #gem 'git' gem 'github-markup' gem 'jekyll' -gem 'jekyll-relative-links' gem 'jekyll-sitemap' gem 'liquid'#, '~> 2.6.2' gem 'mercurial-ruby' diff --git a/README.md b/README.md index d611995..0390b4e 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ Additionally, some make targets are provided for convenience: - To skip everything but repo discovering: ```bash - make discover # + make discover ``` - To skip everything but repo updates: diff --git a/_config.yml b/_config.yml index 071fe1d..1ecc601 100644 --- a/_config.yml +++ b/_config.yml @@ -101,9 +101,5 @@ repos_per_page: 50 # uncomment the following line for testing (limit the number of indexed repos) max_repos: 00 -readme_index: - remove_originals: true - plugins: - jekyll-sitemap - - jekyll-relative-links diff --git a/_layouts/doc.html b/_layouts/doc.html index e81d07d..5002f0b 100644 --- a/_layouts/doc.html +++ b/_layouts/doc.html @@ -6,34 +6,19 @@
    -
  1. Home
    2. - {% assign splitted_url = page.url | split:"/" %} - {% assign repo_name = splitted_url[2] %} - {% assign page_name = splitted_url[-1] %} - - {% assign description = site.docs_repos[repo_name].description %} - - {% if repo_name != page_name %} -
  2. {{ description }}
    3. - {% if page.title == "" or page.title == nil %} -
  3. {{ page_name | remove: ".html" }}
    4. - {% else %} -
  4. {{ page.title }}
    5. - {% endif %} - {% else %} -
  5. {{ description }}
    6. - {% endif %} +
  6. Home
    7. +
  7. {{ page.docs_title }}
    8. + {% unless page.at_root %} +
  8. {{ page.title }}
    9. + {% endunless %}
- {% if page.edit_url %} +
+
Edit on Github - {% endif %} -
-
-
- {{ content }} -
+ {{ content }} +  
diff --git a/_plugins/docs_generator.rb b/_plugins/docs_generator.rb index 6030dea..9e1e8d5 100644 --- a/_plugins/docs_generator.rb +++ b/_plugins/docs_generator.rb @@ -1,3 +1,4 @@ +require 'addressable' require 'fileutils' require 'nokogiri' @@ -17,94 +18,91 @@ def initialize(config = {}) end def generate(site) - doc_repos = site.config['docs_repos'] - docfiles = search_docfiles(doc_repos) - copy_docs_to_sphinx_location(docfiles) - docs_data = convert_with_sphinx(docfiles) - docs_data.each do |repo_name, repo_docs| - repo_docs.each do |key, doc_data| - site.pages << DocPage.new(site, repo_name, doc_data) + all_repos = site.data['remotes']['repositories'] + site.config['docs_repos'].each do |repo_name, repo_options| + next unless all_repos.key? repo_name + repo_data = all_repos[repo_name] + repo_description = repo_options['description'] || repo_name + convert_with_sphinx(repo_name, repo_data).each do |path, content| + site.pages << DocPage.new( + site, repo_name, repo_description, path, content + ) end end end - def copy_docs_to_sphinx_location(docfiles) - docfiles.each do |repo_name, files| - origin_dir = File.join("_remotes", repo_name) - destination_dir = File.join("_sphinx", "repos", repo_name) - unless File.directory?(destination_dir) - FileUtils.makedirs(destination_dir) - end - files.each do |file| - if file.scan(/readme/i).length > 0 - dest_name = "index" + File.extname(file) - else - dest_name = File.basename(file) - end - FileUtils.copy_entry(file, File.join(destination_dir, dest_name), preserve = true) + def copy_docs(src_path, dst_path) + copied_docs = Hash.new + src_path = Pathname.new(src_path) + Dir.glob(File.join(src_path, '**/*.rst'), + File::FNM_CASEFOLD).each do |src_doc_path| + src_doc_path = Pathname.new(src_doc_path) + dst_doc_path = Pathname.new(File.join( + dst_path, src_doc_path.relative_path_from(src_path) + ).sub(/readme\.rst$/i, 'index.rst')) + unless File.directory? File.dirname(dst_doc_path) + FileUtils.makedirs(File.dirname(dst_doc_path)) end + FileUtils.copy_entry(src_doc_path, dst_doc_path, preserve = true) + copied_docs[src_doc_path] = dst_doc_path end + copied_docs end - def search_docfiles(doc_repos_hash) - docfiles_hash = Hash.new - doc_repos_hash.each do |repo_name, repo| - docs_in_repo = Dir.glob(File.join("_remotes", repo_name) + '**/*.{md, rst}', File::FNM_CASEFOLD) - docfiles_hash[repo_name] = docs_in_repo + def generate_edit_url(repo_data, original_filepath) + is_https = repo_data['url'].include? "https" + is_github = repo_data['url'].include? "github.com" + is_bitbucket = repo_data['url'].include? "bitbucket.org" + unless is_github or is_bitbucket + raise ValueError("Cannot generate edition URL. Unknown organization for repository: #{repo_data['url']}") + end + if is_https + uri = URI(repo_data['url']) + host = uri.host + organization, repo = uri.path.split("/").reject { |c| c.empty? } + else # ssh + host, path = repo_data['url'].split("@")[1].split(":") + organization, repo = path.split("/") + end + repo.chomp!(".git") if repo.end_with? ".git" + if is_github + edit_url = "https://#{host}/#{organization}/#{repo}/edit/#{repo_data['version']}" + return File.join(edit_url, original_filepath) + elsif is_bitbucket + edit_url = "https://#{host}/#{organization}/#{repo}/src/#{repo_data['version']}" + return File.join(edit_url, original_filepath) + + "?mode=edit&spa=0&at=#{repo_data['version']}&fileviewer=file-view-default" end - docfiles_hash end - def convert_with_sphinx(docfiles) - json_files_data = Hash.recursive - docfiles.each do |repo_name, files| - repo_path = File.join('_sphinx', 'repos', repo_name) - command = "sphinx-build -b json -c _sphinx #{repo_path} _sphinx/_build/#{repo_name}" - pid = Kernel.spawn(command) - Process.wait pid - - files.each do |file| - leading_single_quotation = /^'/ - trailing_single_quotation = /'$/ - file.sub!(/#{leading_single_quotation} | #{trailing_single_quotation}/, '') - name = File.basename(file, File.extname(file)) - if name.scan(/readme/i).length > 0 - name = "index" - end - json_file = File.join('_sphinx/_build/', repo_name, name + '.fjson') - next unless File.file?(json_file) - json_content = File.read(json_file) - json_files_data[repo_name][name] = JSON.parse(json_content) - json_files_data[repo_name][name]["relative_path"] = file - - # Generate HTML tables that were left with markdown syntax by sphinx. - # Please refer to this issue for more information: - # https://github.com/rtfd/recommonmark/issues/3 - # Although there's a sphinx-markdown-tables extension now, its - # functionality is limited and doesn't fit our requirements. - html_doc = Nokogiri::HTML(JSON.parse(json_content)["body"]) - html_doc.css('p').each do |paragraph| - if paragraph.content.strip[0] == "|" - html_table = Kramdown::Document.new(paragraph.inner_html).to_html - nokogiri_table = Nokogiri::HTML(html_table) - next unless nokogiri_table.css('td').length > 0 - nokogiri_table.css('table').each do |table| - # Fixes table by removing the second row that's always filled - # with hyphens, as a consequence of conversion. - table.css('tr').each_with_index do |tr, index| - if index == 1 - tr.remove - break - end - end - table.set_attribute("class", "table table-striped table-dark") - end - paragraph.replace(nokogiri_table.to_html) - end - end - json_files_data[repo_name][name]["body"] = html_doc.css('body').first.inner_html + def convert_with_sphinx(repo_name, repo_data) + repo_path = Pathname.new(File.join("_remotes", repo_name)) + in_path = Pathname.new(File.join('_sphinx', 'repos', repo_name)) + FileUtils.rm_r(in_path) if File.directory? in_path + copied_docs_paths = copy_docs(repo_path, in_path) + return if copied_docs_paths.empty? + out_path = Pathname.new(File.join('_sphinx', '_build', repo_name)) + FileUtils.rm_r(out_path) if File.directory? out_path + FileUtils.makedirs(out_path) + command = "sphinx-build -b json -c _sphinx #{in_path} #{out_path}" + pid = Kernel.spawn(command) + Process.wait pid + repo_content = Hash.recursive + copied_docs_paths.each do |src_path, dst_path| + json_path = File.join( + out_path, dst_path.relative_path_from(in_path) + ).sub(File.extname(dst_path), '.fjson') + next unless File.file? json_path + json_content = JSON.parse(File.read(json_path)) + json_content["edit_url"] = generate_edit_url( + repo_data, src_path.relative_path_from(repo_path) + ) + permalink = json_content["current_page_name"] + if File.basename(permalink) == "index" + permalink = File.dirname(permalink) end + repo_content[permalink] = json_content end - json_files_data + repo_content end end diff --git a/_plugins/editable_docs.rb b/_plugins/editable_docs.rb deleted file mode 100644 index 3a81b6e..0000000 --- a/_plugins/editable_docs.rb +++ /dev/null @@ -1,43 +0,0 @@ -Jekyll::Hooks.register :site, :pre_render do |site, payload| - repositories = site.data["remotes"]["repositories"] - site.pages.each do |page| - page_path = page.path - if page.data.key?("origin_path") - page_path = page.data["origin_path"] - end - next unless page_path.start_with?("doc") - repo_name, repo_data = repositories.find do |name, data| - page_path.start_with?(File.join("doc", name)) - end - next if repo_name.nil? or repo_data.nil? - # Generate edit url - file_relpath = page.data["file_relpath"] - file_relpath_to_repo = file_relpath.slice((file_relpath.index(repo_name) + repo_name.length)..-1) - page.data["edit_url"] = generate_edit_url(repo_data, file_relpath_to_repo) - end -end - -def generate_edit_url(repo_data, original_filepath) - is_https = repo_data['url'].include? "https" - is_github = repo_data['url'].include? "github.com" - is_bitbucket = repo_data['url'].include? "bitbucket.org" - unless is_github or is_bitbucket - raise ValueError("Cannot generate edition URL. Unknown organization for repository: #{repo_data['url']}") - end - if is_https - uri = URI(repo_data['url']) - host = uri.host - organization, repo = uri.path.split("/").reject { |c| c.empty? } - else # ssh - host, path = repo_data['url'].split("@")[1].split(":") - organization, repo = path.split("/") - end - repo.chomp!(".git") if repo.end_with? ".git" - if is_github - edit_url = "https://#{host}/#{organization}/#{repo}/edit/#{repo_data['version']}" - return File.join(edit_url, original_filepath) - elsif is_bitbucket - edit_url = "https://#{host}/#{organization}/#{repo}/src/#{repo_data['version']}" - return File.join(edit_url, original_filepath) + "?mode=edit&spa=0&at=#{repo_data['version']}&fileviewer=file-view-default" - end -end diff --git a/_ruby_libs/pages.rb b/_ruby_libs/pages.rb index 3751e1c..e45be69 100644 --- a/_ruby_libs/pages.rb +++ b/_ruby_libs/pages.rb @@ -172,23 +172,22 @@ def initialize(site, sort_id, n_list_pages, page_index, list, default=false) end class DocPage < Jekyll::Page - def initialize(site, repo_name, page_data) - basepath = File.join('doc', repo_name) + def initialize(site, docs_name, docs_title, page_name, page_data) @site = site @base = site.source + @dir = "doc/#{docs_name}/#{page_name}" @name = "index.html" - if page_data["current_page_name"].scan(/index|readme/i).length > 0 - @dir = "doc/#{repo_name}/" - else - @dir = "doc/#{repo_name}/#{page_data["current_page_name"]}/" - end self.process(@name) self.data ||= {} + self.content = page_data['body'] self.data['layout'] = "doc" - self.content = page_data["body"] - self.data['file_extension'] = page_data["page_source_suffix"] - self.data['file_relpath'] = page_data["relative_path"] - self.data['title'] = page_data["current_page_name"] + self.data['at_root'] = (page_name == '.') + self.data['title'] = File.basename( + page_name != '.' ? page_name.gsub('-', ' ') : docs_title + ) + self.data['edit_url'] = page_data['edit_url'] + self.data['docs_baseurl'] = "#{site.baseurl}/doc/#{docs_name}" + self.data['docs_title'] = docs_title end end diff --git a/_sphinx/_static/.gitignore b/_sphinx/_static/.gitignore deleted file mode 100644 index d6b7ef3..0000000 --- a/_sphinx/_static/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/_sphinx/conf.py b/_sphinx/conf.py index a1db60c..a69a555 100644 --- a/_sphinx/conf.py +++ b/_sphinx/conf.py @@ -18,19 +18,20 @@ # # import os # sys.path.insert(0, os.path.abspath('.')) -import sys -import os -import recommonmark -from recommonmark.transform import AutoStructify # The suffix(es) of source filenames. # You can specify multiple suffix as a list of string: # -source_suffix = ['.rst', '.md'] # The master toctree document. master_doc = 'index' +# The default role +default_role = 'any' + +# The set of warnings to suppress. +suppress_warnings = ['image.nonlocal_uri'] + # General information about the project. project = u'rosindex' copyright = u'2018, Open Robotics' @@ -66,6 +67,21 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = False +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +extensions = ['sphinx.ext.intersphinx'] + +# Intersphinx mapping + +intersphinx_mapping = { + 'catkin_pkg': ('http://docs.ros.org/independent/api/catkin_pkg/html', None), + 'jenkins_tools': ('http://docs.ros.org/independent/api/jenkins_tools/html', None), + 'rosdep': ('http://docs.ros.org/independent/api/rosdep/html', None), + 'rosdistro': ('http://docs.ros.org/independent/api/rosdistro/html', None), + 'rosinstall': ('http://docs.ros.org/independent/api/rosinstall/html', None), + 'rospkg': ('http://docs.ros.org/independent/api/rospkg/html', None), + 'vcstools': ('http://docs.ros.org/independent/api/vcstools/html', None) +} # -- Options for HTML output ---------------------------------------------- @@ -77,16 +93,9 @@ # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +# html_static_path = ['_static'] # -- Options for HTMLHelp output ------------------------------------------ # Output file base name for HTML help builder. htmlhelp_basename = 'ros2_docsdoc' - -source_parsers = { - '.md': 'recommonmark.parser.CommonMarkParser', -} - -def setup(app): - app.add_transform(AutoStructify)