Merge pull request #1853 from foo-dogsquared/update-godot-scraper

Add Godot v3.4 and v3.5

assets/javascripts/templates/pages/about_tmpl.coffee

@@ -372,7 +372,7 @@ credits = [
     'https://creativecommons.org/licenses/by/3.0/'
   ], [
     'Godot',
-    '2014-2021 Juan Linietsky, Ariel Manzur, Godot Engine contributors',
+    '2014-2022 Juan Linietsky, Ariel Manzur, Godot Engine contributors',
     'MIT',
     'https://raw.githubusercontent.com/godotengine/godot/master/LICENSE.txt'
   ], [

lib/docs/filters/godot/clean_html.rb

@@ -4,7 +4,7 @@ class CleanHtmlFilter < Filter
       def call
         if root_page?
           at_css('h1').content = 'Godot Engine'
-          at_css('.admonition.tip').remove
+          at_css('.admonition.note').remove
         end
 
         css('ul[id].simple li:first-child:last-child').each do |node|

lib/docs/filters/godot/clean_html_v2.rb

@@ -0,0 +1,27 @@
+module Docs
+  class Godot
+    class CleanHtmlV2Filter < Filter
+      def call
+        if root_page?
+          at_css('h1').content = 'Godot Engine'
+          at_css('.admonition.tip').remove
+        end
+
+        css('ul[id].simple li:first-child:last-child').each do |node|
+          heading = Nokogiri::XML::Node.new 'h3', doc.document
+          heading['id'] = node.parent['id']
+          heading.children = node.children
+          node.parent.before(heading).remove
+        end
+
+        css('h3 strong').each do |node|
+          node.before(node.children).remove
+        end
+
+        css('a.reference').remove_attr('class')
+
+        doc
+      end
+    end
+  end
+end

lib/docs/filters/godot/entries.rb

@@ -7,17 +7,11 @@ def get_name
         name
       end
 
-      TYPE_BY_LEARNING_PATH = {
-        'step_by_step' => 'Guides: Step by step',
-        'editor' => 'Guides: Editor',
-        'features' => 'Guides: Engine features',
-        'scripting' => 'Guides: Scripting',
-        'workflow' => 'Guides: Project workflow'
-      }
-
       def get_type
-        if slug.start_with?('learning')
-          TYPE_BY_LEARNING_PATH[slug.split('/')[1]]
+        if slug.start_with?('getting_started')
+          # Getting started sections are different even between different minor
+          # versions from v3 so we're programmatically generating them instead.
+          "Getting started: " + slug.split('/')[1].tr_s('_', ' ').capitalize
         else
           name
         end
@@ -36,7 +30,7 @@ def additional_entries
       end
 
       def include_default_entry?
-        return false if subpath.start_with?('learning') && subpath.end_with?('index.html')
+        return false if subpath.start_with?('getting_started') && subpath.end_with?('index.html')
         return false if subpath == 'classes/index.html'
         true
       end

lib/docs/filters/godot/entries_v2.rb

@@ -0,0 +1,45 @@
+module Docs
+  class Godot
+    class EntriesV2Filter < Docs::EntriesFilter
+      def get_name
+        name = at_css('h1').content
+        name.remove! "\u{00B6}" # Remove the pilcrow
+        name
+      end
+
+      TYPE_BY_LEARNING_PATH = {
+        'step_by_step' => 'Guides: Step by step',
+        'editor' => 'Guides: Editor',
+        'features' => 'Guides: Engine features',
+        'scripting' => 'Guides: Scripting',
+        'workflow' => 'Guides: Project workflow'
+      }
+
+      def get_type
+        if slug.start_with?('learning')
+          TYPE_BY_LEARNING_PATH[slug.split('/')[1]]
+        else
+          name
+        end
+      end
+
+      def additional_entries
+        return [] unless slug.start_with?('classes')
+
+        css('.simple[id]').each_with_object [] do |node, entries|
+          name = node.at_css('strong').content
+          next if name == self.name
+          name.prepend "#{self.name}."
+          name << '()'
+          entries << [name, node['id']] unless entries.any? { |entry| entry[0] == name }
+        end
+      end
+
+      def include_default_entry?
+        return false if subpath.start_with?('learning') && subpath.end_with?('index.html')
+        return false if subpath == 'classes/index.html'
+        true
+      end
+    end
+  end
+end

lib/docs/scrapers/godot.rb

@@ -6,41 +6,59 @@ class Godot < UrlScraper
       code: 'https://github.com/godotengine/godot'
     }
 
-    html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
-
-    options[:download_images] = false
     options[:container] = '.document .section'
 
-    options[:only_patterns] = [/\Alearning\//, /\Aclasses\//]
-    options[:skip] = %w(classes/class_@global\ scope.html)
+    options[:download_images] = false
+    options[:only_patterns] = [/\Agetting_started\//, /\Aclasses\//]
 
     options[:attribution] = ->(filter) do
       if filter.subpath.start_with?('classes')
          <<-HTML
-          &copy; 2014&ndash;2021 Juan Linietsky, Ariel Manzur, Godot Engine contributors<br>
+          &copy; 2014&ndash;2022 Juan Linietsky, Ariel Manzur, Godot Engine contributors<br>
           Licensed under the MIT License.
         HTML
       else
         <<-HTML
-          &copy; 2014&ndash;2021 Juan Linietsky, Ariel Manzur and the Godot community<br>
+          &copy; 2014&ndash;2022 Juan Linietsky, Ariel Manzur and the Godot community<br>
           Licensed under the Creative Commons Attribution Unported License v3.0.
         HTML
       end
     end
 
+    version '3.5' do
+      self.release = '3.5.1'
+      self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
+      options[:container] = '.document > [itemprop="articleBody"] > section[id]'
+      html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
+    end
+
+    version '3.4' do
+      self.release = '3.4.5'
+      self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
+      options[:container] = '.document > [itemprop="articleBody"] > section[id]'
+      html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
+    end
+
     version '3.3' do
       self.release = '3.3.0'
       self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
+      html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
     end
 
     version '3.2' do
       self.release = '3.2.3'
       self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
+      html_filters.push 'godot/entries', 'godot/clean_html', 'sphinx/clean_html'
     end
 
     version '2.1' do
       self.release = '2.1.6'
       self.base_url = "https://docs.godotengine.org/en/#{self.version}/"
+
+      options[:skip] = %w(classes/class_@global\ scope.html)
+      options[:only_patterns] = [/\Alearning\//, /\Aclasses\//]
+
+      html_filters.push 'godot/entries_v2', 'godot/clean_html_v2', 'sphinx/clean_html'
     end
 
     def get_latest_version(opts)