From 20fc5c39d47667652d6b7b5892e95672e4ae6cd1 Mon Sep 17 00:00:00 2001 From: adfoster-r7 Date: Sat, 26 Feb 2022 19:30:51 +0000 Subject: [PATCH 1/5] Add metadata tracking the page that generated the wiki page --- docs/build.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/build.rb b/docs/build.rb index 0ba4ea3079bf..3c4084839709 100644 --- a/docs/build.rb +++ b/docs/build.rb @@ -338,7 +338,9 @@ def run(config, options = {}) page_config = { layout: 'default', **page.slice(:title, :has_children, :nav_order), - parent: (page[:parents][-1] || {})[:title] + parent: (page[:parents][-1] || {})[:title], + warning: "Do not modify this file directly. Please modify metasploit-framework/docs/metasploit-framework.wiki instead", + old_path: page[:path] ? File.join(WIKI_PATH, page[:path]) : "none - folders automatically generated" }.compact page_config[:has_children] = true if page[:has_children] From 255452c5ca354b24887502d4232844a20b9fc8af Mon Sep 17 00:00:00 2001 From: adfoster-r7 Date: Sat, 26 Feb 2022 19:19:54 +0000 Subject: [PATCH 2/5] Update Wiki deprecation notice text, and add functionality to integrate the wiki into framework --- docs/.gitignore | 1 + docs/README.md | 26 ++++++++-- docs/_config.yml | 1 + docs/build.rb | 127 +++++++++++++++++++++++++++++++++++------------ 4 files changed, 118 insertions(+), 37 deletions(-) diff --git a/docs/.gitignore b/docs/.gitignore index d71acd76cd2a..884d15b19fe5 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -7,4 +7,5 @@ vendor # These files will be generated by build.rb and do not need to be committed docs metasploit-framework.wiki +metasploit-framework.wiki.old index.md diff --git a/docs/README.md b/docs/README.md index ef0fc5e3e98e..ee3e0dcb8542 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,6 +1,19 @@ # Metasploit docs site -## Setup +This folder maintains the docs for https://docs.metasploit.com/ and https://github.com/rapid7/metasploit-framework/wiki + +## Architecture + +How it works: + +- `build.rb` - The main entry point for generating the docs site from the old Github Wiki format files within `metasploit-framework.wiki/` +- `navigation.rb` - Stores the mapping of `metasploit-framework.wiki` files to the website's navigational structure +- `metasploit-framework.wiki/` - The raw markdown documentation files. Modify these files when updating the site. These files originally came from https://github.com/rapid7/metasploit-framework/wiki +- `metasploit-framework.wiki.old/` - A separate clone of https://github.com/rapid7/metasploit-framework/wiki + +Behind the scenes these docs are built and deployed to https://docs.metasploit.com/ + +## Setup ### Developer build @@ -32,7 +45,12 @@ Now visit http://127.0.0.1:4000/metasploit-framework/ ### Modifying pages -**Note** - to modify pages, for now, the changes will need to be made to [Metasploit Wiki](https://github.com/rapid7/metasploit-framework/wiki). -However the `docs` folder can be edited locally. Jekyll will rebuild the required file, and the changes can be seen after refreshing your browser. +Modify the files within `metasploit-framework.wiki/`. The `docs` folder is generated, but can be edited locally. +Jekyll will rebuild the required file, and the changes can be seen after refreshing your browser. + +## Adding pages + +Add the new file to `metasploit-framework.wiki/`, as well as adding a new file entry to `navigation.rb` and rebuild the site. -When adding test files locally, Jekyll will not always regenerate the navigation for all pages. It is easier to rebuild the entire site again. +Note that when testing locally - if you're adding new files, Jekyll will not always regenerate the navigation for all pages. +It is easier to rebuild the entire site again. diff --git a/docs/_config.yml b/docs/_config.yml index be3946a4d487..374fcef85269 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -26,6 +26,7 @@ exclude: - '*.rb' - 'build.rb' - metasploit-framework.wiki + - metasploit-framework.wiki.old - README.md # just-the-docs config diff --git a/docs/build.rb b/docs/build.rb index 3c4084839709..fa05fb3bdedc 100644 --- a/docs/build.rb +++ b/docs/build.rb @@ -4,7 +4,7 @@ require 'optparse' require_relative './navigation' -# Temporary build module to help migrate the Metasploit wiki https://github.com/rapid7/metasploit-framework/wiki into a format +# Temporary build module to help migrate and build the Metasploit wiki https://github.com/rapid7/metasploit-framework/wiki into a format # supported by Jekyll, as well as creating a hierarchical folder structure for nested documentation # # For now the doc folder only contains the key files for building the docs site and no content. The content is created on demand @@ -13,7 +13,10 @@ # In the future, the markdown files will be committed directly to the metasploit-framework directory, the wiki history will be # merged with metasploit-framework, and the old wiki will no longer be updated. module Build + # The metasploit-framework.wiki files that are committed to Metasploit framework's repository WIKI_PATH = 'metasploit-framework.wiki'.freeze + # A locally cloned version of https://github.com/rapid7/metasploit-framework/wiki + OLD_WIKI_PATH = 'metasploit-framework.wiki.old'.freeze PRODUCTION_BUILD_ARTIFACTS = '_site'.freeze # For now we Git clone the existing metasploit wiki and generate the Jekyll markdown files @@ -25,13 +28,18 @@ def self.clone_wiki! Build.run_command "git clone https://github.com/rapid7/metasploit-framework.wiki.git #{WIKI_PATH}", exception: true end - Build.run_command "cd #{WIKI_PATH}; git pull", exception: true + unless File.exist?(OLD_WIKI_PATH) + Build.run_command "git clone https://github.com/rapid7/metasploit-framework.wiki.git #{OLD_WIKI_PATH}", exception: true + end + + Build.run_command "cd #{OLD_WIKI_PATH}; git pull", exception: true end end # Configuration for generating the new website hierarchy, from the existing metasploit-framework wiki class Config include Enumerable + def initialize(config) @config = config end @@ -134,10 +142,6 @@ def with_metadata(child, parents: []) child end - def without_prefix(prefix) - proc { |value| value.gsub(/^#{prefix}/, '') } - end - attr_reader :config end @@ -307,16 +311,24 @@ def fix_github_username_links(content) # Parses a wiki page and can add/remove/update a deprecation notice class WikiDeprecationText - MARKDOWN_PREFIX = '#### Documentation Update:'.freeze - private_constant :MARKDOWN_PREFIX + MAINTAINER_MESSAGE_PREFIX = "\n\n" + user_message = "#{USER_MESSAGE_PREFIX} This Wiki page should be viewable at [#{new_url}](#{new_url}). Or if it is no longer available, see this page's [previous history](#{history_link})\n\n" + deprecation_text = (maintainer_message * 3) + user_message + "#{deprecation_text}#{WikiDeprecationText.remove(original_wiki_content)}" end def self.remove(original_wiki_content) - original_wiki_content.gsub(/#{MARKDOWN_PREFIX}.*$\s+/, '') + original_wiki_content + .gsub(/#{MAINTAINER_MESSAGE_PREFIX}.*$\s+/, '') + .gsub(/#{USER_MESSAGE_PREFIX}.*$\s+/, '') end end @@ -340,7 +352,7 @@ def run(config, options = {}) **page.slice(:title, :has_children, :nav_order), parent: (page[:parents][-1] || {})[:title], warning: "Do not modify this file directly. Please modify metasploit-framework/docs/metasploit-framework.wiki instead", - old_path: page[:path] ? File.join(WIKI_PATH, page[:path]) : "none - folders automatically generated" + old_path: page[:path] ? File.join(WIKI_PATH, page[:path]) : "none - folder automatically generated" }.compact page_config[:has_children] = true if page[:has_children] @@ -362,14 +374,15 @@ def run(config, options = {}) new_docs_content = preamble + WikiDeprecationText.remove(previous_content) new_docs_content = link_corrector.rerender(new_docs_content) - # Update the existing Wiki with links to the new website - if options[:update_existing_wiki] - new_url = options[:update_existing_wiki][:new_website_url] + # Update the old Wiki with links to the new website + if options[:update_wiki_deprecation_notice] + new_url = options[:update_wiki_deprecation_notice][:new_website_url] if page[:new_path] != 'home.md' new_url += 'docs/' + page[:new_path].gsub('.md', '.html') end - updated_wiki_content = WikiDeprecationText.upsert(previous_content, new_url: new_url) - File.write(old_path, updated_wiki_content) + updated_wiki_content = WikiDeprecationText.upsert(previous_content, old_path: old_path, new_url: new_url) + old_wiki_path = File.join(OLD_WIKI_PATH, page[:path]) + File.write(old_wiki_path, updated_wiki_content, mode: 'w', encoding: Encoding::UTF_8) end end @@ -450,9 +463,50 @@ def self.run_command(command, exception: true) end def self.run(options) - Git.clone_wiki! unless options[:skip_wiki_pull] + Git.clone_wiki! if options[:wiki_pull] + + # Create a new branch based on the commits from https://github.com/rapid7/metasploit-framework/wiki to move + # Wiki files into the metasploit-framework repo + if options[:create_wiki_to_framework_migration_branch] + starting_branch = run_command("git rev-parse --abbrev-ref HEAD").chomp + new_wiki_branch_name = "move-all-docs-into-folder" + new_framework_branch_name = "merge-metasploit-framework-wiki-into-metasploit-framework" + + begin + # Create a new folder and branch in the old metasploit wiki for where we'd like it to be inside of the metasploit-framework repo + Dir.chdir(OLD_WIKI_PATH) do + FileUtils.mkdir_p("metasploit-framework.wiki") + run_command("mv *[^metasploit-framework.wiki]* metasploit-framework.wiki", exception: false) + run_command("git branch -d #{new_wiki_branch_name}", exception: false) + run_command("git checkout -b #{new_wiki_branch_name}") + run_command("git commit -am 'Move to folder'") + end + + # Create a new branch that can be used to create a pull request + run_command("git branch -D #{new_framework_branch_name}", exception: false) + run_command("git checkout -b #{new_framework_branch_name}") + run_command("git remote add -f wiki #{OLD_WIKI_PATH}", exception: false) + run_command("git remote update wiki") + run_command("git merge -m 'Migrate docs from wiki to main repository' wiki/#{new_wiki_branch_name} --allow-unrelated-histories") + + puts "new branch #{new_framework_branch_name} successfully created" + ensure + run_command("git checkout #{starting_branch}") + end + end + + if options[:copy_old_wiki] + FileUtils.copy_entry(OLD_WIKI_PATH, WIKI_PATH, preserve = false, dereference_root = false, remove_destination = true) + # Remove any deprecation text that might be present after copying the old wiki + Dir.glob(File.join(WIKI_PATH, '**', '*.md')) do |path| + previous_content = File.read(path, encoding: Encoding::UTF_8) + new_content = WikiDeprecationText.remove(previous_content) + + File.write(path, new_content, mode: 'w', encoding: Encoding::UTF_8) + end + end - unless options[:skip_migration] + unless options[:build_content] config = Config.new(NAVIGATION_CONFIG) migrator = WikiMigration.new migrator.run(config, options) @@ -472,7 +526,10 @@ def self.run(options) end if $PROGRAM_NAME == __FILE__ - options = {} + options = { + copy_old_wiki: true, + wiki_pull: true + } options_parser = OptionParser.new do |opts| opts.banner = "Usage: #{File.basename(__FILE__)} [options]" @@ -480,27 +537,31 @@ def self.run(options) return print(opts.help) end - opts.on('--skip-wiki-pull', 'Skip pulling the Metasploit Wiki') do |skip_wiki_pull| - options[:skip_wiki_pull] = skip_wiki_pull + opts.on('--production', 'Run a production build') do |production| + options[:production] = production + end + + opts.on('--serve', 'serve the docs site') do |serve| + options[:serve] = serve + end + + opts.on('--[no]-copy-old-wiki [FLAG]', TrueClass, 'Copy the content from the old wiki to the new local wiki folder') do |copy_old_wiki| + options[:copy_old_wiki] = copy_old_wiki end - opts.on('--skip-migration', 'Skip building the content') do |skip_migration| - options[:skip_migration] = skip_migration + opts.on('--[no-]-wiki-pull', FalseClass, 'Pull the Metasploit Wiki') do |wiki_pull| + options[:wiki_pull] = wiki_pull end - opts.on('--update-existing-wiki [website url]', 'Update the existing wiki with links to the new website location') do |new_website_url| + opts.on('--update-wiki-deprecation-notice [WEBSITE_URL]', 'Updates the old wiki deprecation notes') do |new_website_url| new_website_url ||= 'https://docs.metasploit.com/' - options[:update_existing_wiki] = { + options[:update_wiki_deprecation_notice] = { new_website_url: new_website_url } end - opts.on('--production', 'Run a production build') do |production| - options[:production] = production - end - - opts.on('--serve', 'serve the docs site') do |serve| - options[:serve] = serve + opts.on('--create-wiki-to-framework-migration-branch') do + options[:create_wiki_to_framework_migration_branch] = true end end options_parser.parse! From 669d95ba168017d620cd782302116143bf55469c Mon Sep 17 00:00:00 2001 From: adfoster-r7 Date: Wed, 18 May 2022 13:37:30 +0100 Subject: [PATCH 3/5] Ensure files are correctly added to the migrated wiki --- docs/build.rb | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/build.rb b/docs/build.rb index fa05fb3bdedc..509484abb9f5 100644 --- a/docs/build.rb +++ b/docs/build.rb @@ -431,7 +431,7 @@ def self.run end def self.run_command(command, exception: true) - puts command + puts "[*] #{command}" result = '' ::Open3.popen2e( { 'BUNDLE_GEMFILE' => File.join(Dir.pwd, 'Gemfile') }, @@ -455,7 +455,7 @@ def self.run_command(command, exception: true) end if !wait_thread.value.success? && exception - raise "command did not succeed, exit status #{wait_thread.value.exitstatus.inspect}" + raise "command #{command.inspect} did not succeed, exit status #{wait_thread.value.exitstatus.inspect}" end end @@ -475,19 +475,29 @@ def self.run(options) begin # Create a new folder and branch in the old metasploit wiki for where we'd like it to be inside of the metasploit-framework repo Dir.chdir(OLD_WIKI_PATH) do + # Reset the repo back + run_command("git checkout master", exception: false) + run_command("git reset HEAD --hard", exception: false) + run_command("rm -rf metasploit-framework.wiki", exception: false) + + # Create a new folder to move the wiki contents into FileUtils.mkdir_p("metasploit-framework.wiki") run_command("mv *[^metasploit-framework.wiki]* metasploit-framework.wiki", exception: false) - run_command("git branch -d #{new_wiki_branch_name}", exception: false) + + # Create a new branch + commit + run_command("git branch -D #{new_wiki_branch_name}", exception: false) run_command("git checkout -b #{new_wiki_branch_name}") - run_command("git commit -am 'Move to folder'") + run_command("git add metasploit-framework.wiki") + run_command("git commit -am 'Put markdown files into new folder metasploit-framework.wiki in preparation for migration'") end # Create a new branch that can be used to create a pull request run_command("git branch -D #{new_framework_branch_name}", exception: false) run_command("git checkout -b #{new_framework_branch_name}") - run_command("git remote add -f wiki #{OLD_WIKI_PATH}", exception: false) - run_command("git remote update wiki") - run_command("git merge -m 'Migrate docs from wiki to main repository' wiki/#{new_wiki_branch_name} --allow-unrelated-histories") + run_command("git remote remove wiki", exception: false) + run_command("git remote add -f wiki #{File.join(Dir.pwd, OLD_WIKI_PATH)}", exception: false) + # run_command("git remote update wiki") + run_command("git merge -m 'Migrate docs from https://github.com/rapid7/metasploit-framework/wiki to main repository' wiki/#{new_wiki_branch_name} --allow-unrelated-histories") puts "new branch #{new_framework_branch_name} successfully created" ensure From 10d325761fc621dc76c4bf952425ad300f476b28 Mon Sep 17 00:00:00 2001 From: adfoster-r7 Date: Wed, 18 May 2022 22:32:20 +0100 Subject: [PATCH 4/5] Remove deprecation notice title as it appears in the submenu links --- docs/build.rb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/build.rb b/docs/build.rb index 509484abb9f5..3c945aa25c4c 100644 --- a/docs/build.rb +++ b/docs/build.rb @@ -314,21 +314,21 @@ class WikiDeprecationText MAINTAINER_MESSAGE_PREFIX = "\n\n" - user_message = "#{USER_MESSAGE_PREFIX} This Wiki page should be viewable at [#{new_url}](#{new_url}). Or if it is no longer available, see this page's [previous history](#{history_link})\n\n" - deprecation_text = (maintainer_message * 3) + user_message - "#{deprecation_text}#{WikiDeprecationText.remove(original_wiki_content)}" + user_message = "#{USER_MESSAGE_PREFIX} This Wiki page should be viewable at [#{new_url}](#{new_url}). Or if it is no longer available, see this page's [previous history](#{history_link})**\n\n" + deprecation_text = maintainer_message + user_message + "#{deprecation_text}" end def self.remove(original_wiki_content) original_wiki_content - .gsub(/#{MAINTAINER_MESSAGE_PREFIX}.*$\s+/, '') - .gsub(/#{USER_MESSAGE_PREFIX}.*$\s+/, '') + .gsub(/^#{Regexp.escape(MAINTAINER_MESSAGE_PREFIX)}.*$\s+/, '') + .gsub(/^#{Regexp.escape(USER_MESSAGE_PREFIX)}.*$\s+/, '') end end From eea4c17dca18727371b03b3a7ba6c95a4bf56ccf Mon Sep 17 00:00:00 2001 From: adfoster-r7 Date: Thu, 19 May 2022 16:10:47 +0100 Subject: [PATCH 5/5] Simplify contributing steps --- docs/README.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/docs/README.md b/docs/README.md index ee3e0dcb8542..2657bc42cbd2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -43,14 +43,8 @@ bundle exec ruby build.rb --production --serve Now visit http://127.0.0.1:4000/metasploit-framework/ -### Modifying pages -Modify the files within `metasploit-framework.wiki/`. The `docs` folder is generated, but can be edited locally. -Jekyll will rebuild the required file, and the changes can be seen after refreshing your browser. +### Contributing Documentation -## Adding pages - -Add the new file to `metasploit-framework.wiki/`, as well as adding a new file entry to `navigation.rb` and rebuild the site. - -Note that when testing locally - if you're adding new files, Jekyll will not always regenerate the navigation for all pages. -It is easier to rebuild the entire site again. +You can modify existing documentation files within `metasploit-framework.wiki/` with an editor of your choice and send a pull request. +Note that adding a new page will also require modifying `navigation.rb` to appear on the navigation menu.