WIP import script bbcode fixes in Fluxbb importer + some fixes to all importers

Gemfile

@@ -253,7 +253,7 @@ if ENV["IMPORT"] == "1"
 
   # NOTE: in import mode the version of sqlite can matter a lot, so we stick it to a specific one
   gem "sqlite3", "~> 1.3", ">= 1.3.13"
-  gem "ruby-bbcode-to-md", git: "https://github.com/nlalonde/ruby-bbcode-to-md"
+  gem 'ruby-bbcode-to-md', git: 'https://github.com/harry-wood/ruby-bbcode-to-md', branch: 'fluxbb-tests-and-fixes'
   gem "reverse_markdown"
   gem "tiny_tds"
   gem "csv"

script/import_scripts/base.rb

@@ -126,6 +126,10 @@ def use_bbcode_to_md?
     ARGV.include?("bbcode-to-md")
   end
 
+  def bbcode_tag_additions_and_overrides
+    {}
+  end
+
   # Implementation will do most of its work in its execute method.
   # It will need to call create_users, create_categories, and create_posts.
   def execute
@@ -606,7 +610,7 @@ def create_post(opts, import_id)
     opts[:guardian] = STAFF_GUARDIAN
     if @bbcode_to_md
       opts[:raw] = begin
-        opts[:raw].bbcode_to_md(false, {}, :disable, :quote)
+        opts[:raw].bbcode_to_md(false, bbcode_tag_additions_and_overrides, :disable, :quote, :u)
       rescue StandardError
         opts[:raw]
       end

script/import_scripts/fluxbb.rb

@@ -24,6 +24,12 @@ class ImportScripts::FluxBB < ImportScripts::Base
   FLUXBB_PW ||= ENV["FLUXBB_PW"] || ""
   FLUXBB_PREFIX ||= ENV["FLUXBB_PREFIX"] || ""
 
+  # Set this if you want to rewrite relative links to be absolute.
+  # e.g. "https://mysite.com/old-forum-redirector/"
+  FLUXBB_RELATIVE_LINKS_BASE ||= ENV['FLUXBB_RELATIVE_LINKS_BASE'] || ""
+
+  CREATE_PERMALINKS = true
+
   def initialize
     super
 
@@ -42,6 +48,7 @@ def execute
     import_categories
     import_posts
     suspend_users
+    create_permalinks if CREATE_PERMALINKS
   end
 
   def import_groups
@@ -236,6 +243,32 @@ def suspend_users
   def process_fluxbb_post(raw, import_id)
     s = raw.dup
 
+    unless FLUXBB_RELATIVE_LINKS_BASE.blank?
+      s.gsub!(/\[url\]\s*\//i, "[url]#{FLUXBB_RELATIVE_LINKS_BASE}")
+      s.gsub!(/\[url=\s*\/(.*)\s*\]/i, "[url=#{FLUXBB_RELATIVE_LINKS_BASE}\\1]")
+    end
+
+    # FluxBB supports slightly different smiley character combos vs discourse
+    s.gsub!(/(\s)=\)/, "\\1:)")
+    s.gsub!(/(\s)=\|/, "\\1:|")
+    s.gsub!(/(\s)=\(/, "\\1:(")
+    s.gsub!(/(\s)=D/, "\\1:D")
+    s.gsub!(/(\s):o/, "\\1:O")
+    s.gsub!(/(\s):lol:/, '\\1:laughing:')
+    s.gsub!(/(\s):mad:/, '\\1:rage:')
+    s.gsub!(/(\s):rolleyes:/, '\\1:roll_eyes:')
+
+    # Inside [list] tags convert [*][/*] to <li></li>
+    s.gsub!(/\[list(?:=.)?\](.*?)\[\/list\]/im) do |list_contents|
+      list_contents
+        .gsub(/\[\*\](.*?)\[\/\*\]/m, "<li>\n\n\\1</li>")
+        .gsub(/\[\*\]/m, "<li>\n\n") # Unclosed [*] are also allowed
+    end
+    # convert [list] tags to <ul> and [list=1] tags to <ol>
+    s.gsub!(/\[list\](.*?)\[\/list\]/im, '<ul>\1</ul>')
+    s.gsub!(/\[list=1\](.*?)\[\/list\]/im, '<ol>\1</ol>')
+    s.gsub!(/\[list=a\](.*?)\[\/list\]/im, '<ol class="alpha">\1</ol>')
+
     # :) is encoded as <!-- s:) --><img src="{SMILIES_PATH}/icon_e_smile.gif" alt=":)" title="Smile" /><!-- s:) -->
     s.gsub!(/<!-- s(\S+) -->(?:.*)<!-- s(?:\S+) -->/, '\1')
 
@@ -261,6 +294,120 @@ def process_fluxbb_post(raw, import_id)
     s
   end
 
+  def bbcode_tag_additions_and_overrides
+    link_base = FLUXBB_RELATIVE_LINKS_BASE.blank? ? "/" : FLUXBB_RELATIVE_LINKS_BASE
+
+    {
+      h: {
+        html_open: "\n## ", html_close: "\n",
+        newlines: :to_br,
+        description: 'Make a heading',
+        example: '[h]My Heading[/h]' },
+      del: {
+        html_open: '<del>', html_close: '</del>',
+        description: 'Deleted text',
+        example: 'This is [del]deleted[/del].' },
+      ins: {
+        html_open: '<ins>', html_close: '</ins>',
+        description: 'Inserted text',
+        example: 'This is [ins]inserted[/ins].' },
+      em: {
+        html_open: '**', html_close: '**',
+        description: 'Make text emphasised (same as bold in markdown)',
+        example: 'This is [em]emphasised[/em].' },
+      img: {
+        html_open: '<img src="%between%"%alt_text%/>', html_close: '',
+        description: 'Image',
+        example: '[img=my alt text]http://www.google.com/intl/en_ALL/images/logo.gif[/img].',
+        only_allow: [],
+        require_between: true,
+        allow_tag_param: true, allow_tag_param_between: false,
+        tag_param: /(.*)/,
+        tag_param_tokens: [{ token: :alt_text, prefix: ' alt="', postfix: '"' }],
+        tag_param_description: 'The img bbcode takes alt text as a parameter' },
+      post: {
+        html_open: '[%between%](%post_url%#p%post_id%)', html_close: '',
+        description: 'Link to forum post by id',
+        example: '[post=1]Link text[/post].',
+        only_allow: [],
+        require_between: true,
+        allow_tag_param: true,
+        allow_tag_param_between: true,
+        tag_param: /((\d*))/,
+        tag_param_tokens: [ { token: :post_id },
+                            { token: :post_url, prefix: "#{link_base}viewtopic.php?pid=" }] },
+      topic: {
+        html_open: '[%between%](%topic_url%)', html_close: '',
+        description: 'Link to forum topic by id',
+        example: '[topic=1]Link text[/topic].',
+        only_allow: [],
+        require_between: true,
+        allow_tag_param: true,
+        allow_tag_param_between: true,
+        tag_param: /(\d*)/,
+        tag_param_tokens: [ { token: :topic_url, prefix: "#{link_base}viewtopic.php?id=" }] },
+      forum: {
+        html_open: '[%between%](%forum_url%)', html_close: '',
+        description: 'Link to forum by id',
+        example: '[forum=1]Link text[/forum].',
+        only_allow: [],
+        require_between: true,
+        allow_tag_param: true,
+        allow_tag_param_between: true,
+        tag_param: /(\d*)/,
+        tag_param_tokens: [ { token: :forum_url, prefix: "#{link_base}viewforum.php?id=" }] },
+      user: {
+        html_open: '[%between%](%user_url%)', html_close: '',
+        description: 'Link to user profile by id',
+        example: '[user=1]Link text[/user].',
+        only_allow: [],
+        require_between: true,
+        allow_tag_param: true,
+        allow_tag_param_between: true,
+        tag_param: /(\d*)/,
+        tag_param_tokens: [{ token: :user_url, prefix: "#{link_base}profile.php?id=" }] },
+    }
+  end
+
+  def create_permalinks
+    puts '', 'Creating redirects...', ''
+
+    User.find_each do |user|
+      ucf = user.custom_fields
+      if ucf && ucf["import_id"]
+        old_user_profile_url = "profile.php?id=#{ucf['import_id']}"
+        new_discourse_user_profile_url = "/u/#{user.username}"
+        Permalink.create(url: old_user_profile_url, external_url: new_discourse_user_profile_url)
+      end
+    end
+
+    Post.find_each do |post|
+      pcf = post.custom_fields
+      if pcf && pcf["import_id"]
+        old_post_url = "viewtopic.php?pid=#{pcf["import_id"]}"
+        Permalink.create(url: old_post_url, post_id: post.id)
+      end
+
+      if post.post_number == 1
+        topic = post.topic
+        tcf = topic.custom_fields
+        if tcf && tcf["import_id"]
+          old_topic_url = "viewtopic.php?id=#{tcf['import_id']}"
+          Permalink.create(url: old_topic_url, topic_id: topic.id)
+        end
+      end
+    end
+
+    Category.find_each do |cat|
+      ccf = cat.custom_fields
+      if ccf && ccf["import_id"]
+        old_forum_id = ccf["import_id"].sub("child#", "")
+        old_forum_url = "viewforum.php?id=#{old_forum_id}"
+        Permalink.create(url: old_forum_url, category_id: cat.id)
+      end
+    end
+  end
+
   def mysql_query(sql)
     @client.query(sql, cache_rows: false)
   end