Added sitemap generator to CLI

src/arachnid/cli.cr

@@ -14,24 +14,76 @@ module Arachnid
 
       sub "summarize" do
         desc "Scan a site (or sites) and generate a JSON report"
-        usage "arachnid summarize [sites] [options] ..."
+        usage <<-USAGE
+        arachnid summarize [sites] [options]
+
+          Examples:
+
+            # Scan a site and count the number of pages, outputting the result to STDOUT
+            arachnid summarize https://crystal-lang.org
+
+            # Scan a site and count the number of internal links, outputting the result to STDOUT
+            arachnid summarize https://crystal-lang.org -l
+
+            # Scan a site and count the number of internal and external links, saving the result to a file
+            arachnid summarize https://crystal-lang.org -l -L -o report.json
+
+            # Scan a site and list all pages that returned a 404 or 500 status code
+            arachnid summarize https://crystal-lang.org -c 404 500
+        USAGE
 
         option "-l", "--ilinks",              type: Bool,         desc: "generate a map of pages to internal links"
         option "-L", "--elinks",              type: Bool,         desc: "generate a map of pages to external links"
         option "-c CODES", "--codes=CODES",   type: Array(Int32), desc: "generate a map of status codes to pages \
                                                                           that responded with that code"
         option "-n", "--limit NUM",           type: Int32,        desc: "maximum number of pages to scan"
-        option "-o FILE", "--output=FILE",    type: String,       desc: "file to write the report to", default: "arachnid.json"
+        option "-o FILE", "--output=FILE",    type: String,       desc: "file to write the report to (if undefined \
+                                                                          output will be printed to STDOUT"
 
         run do |opts, args|
-          count = Arachnid::Cli::Count.new
           if args.empty?
             STDERR.puts "At least one site is required"
           else
+            count = Arachnid::Cli::Count.new
             count.run(opts, args)
           end
         end
       end
+
+      sub "sitemap" do
+        desc "generate a sitemap for a site in XML or JSON format"
+        usage <<-USAGE
+        arachnid sitemap [url] [--xml | --json] [options]
+
+          Examples:
+
+            # Generate a XML sitemap for crystal-lang.org
+            arachnid sitemap https://crystal-lang.org --xml
+
+            # Generate a XML sitemap with a custom filename
+            arachnid sitemap https://crystal-lang.org --xml -o ~/Desktop/crystal-lang.org.xml
+
+            # Generate a JSON sitemap instead (not really useful as an actual sitemap)
+            arachnid sitemap https://crystal-lang.org --json
+        USAGE
+
+
+        option "--xml",                     type: Bool, desc: "generate the sitemap in XML format"
+        option "--json",                    type: Bool, desc: "generate the sitemap in JSON format"
+        option "-o FILE", "--output=FILE",  type: String, desc: "filename to write the report to. \
+                                                                default is the hostname + .json or .xml"
+
+        run do |opts, args|
+          if args.size != 1
+            raise "arachnid sitemap requires exactly one site to scan. you provided #{args.size}"
+          elsif !opts.json && !opts.xml
+            raise "you must select either xml or json"
+          else
+            sitemap = Arachnid::Cli::Sitemap.new
+            sitemap.run(opts, args)
+          end
+        end
+      end
     end
   end
 end

src/arachnid/cli/count.cr

@@ -64,8 +64,12 @@ module Arachnid
         report["external_links"] = external_links if external_links
         report["codes"] = codes if codes
 
-        File.write(outfile, report.to_json, mode: "w+")
-        puts "Report saved to #{outfile}"
+        if outfile
+          File.write(outfile.to_s, report.to_json, mode: "w+")
+          puts "Report saved to #{outfile}"
+        else
+          pp report
+        end
       end
     end
   end

src/arachnid/cli/sitemap.cr

@@ -0,0 +1,90 @@
+require "./action"
+require "../arachnid"
+require "termspinner"
+require "json"
+
+module Arachnid
+  class Cli < Clim
+    class Sitemap < Cli::Action
+
+      alias LastMod = NamedTuple(year: String, month: String, day: String)
+      alias PageMap = NamedTuple(url: String, page: String, changefreq: String, priority: String, lastmod: LastMod)
+
+      def run(opts, args)
+        url = URI.parse(args[0])
+        date = Time.now
+        spinner = Spinner::Spinner.new("Wait...")
+
+        spider = Arachnid::Agent.new
+        spider.visit_urls_like(Regex.new(url.to_s))
+
+        map = {
+          domain: url.to_s,
+          lastmod: {
+            year: date.year.to_s, month: date.month.to_s, day: date.day.to_s
+          },
+          filetype: "html",
+
+          pages: [] of PageMap
+        }
+
+        spinner.start("Crawling...")
+
+        spider.every_html_page do |page|
+          spinner.message = "Crawling... Current page #{page.url.to_s}"
+          last_mod = page.headers["Last-Modified"]?
+          last_mod = last_mod ? Time.parse_utc(last_mod, "%a, %d %b %Y %H:%M:%S GMT") : Time.now
+
+          item = {
+            url: page.url.to_s,
+            page: page.title.to_s,
+            changefreq: "never",
+            priority: "0.5",
+            lastmod: {
+              year: last_mod.year.to_s,
+              month: last_mod.month.to_s.rjust(2, '0'),
+              day: last_mod.day.to_s.rjust(2, '0')
+            }
+          }
+
+          map[:pages] << item
+        end
+
+        spider.start_at(url)
+        spinner.stop("Finished scanning!\n")
+
+        if opts.xml
+          filename = (opts.output ? opts.output.to_s : url.hostname.to_s + ".xml")
+          sitemap = gen_xml_sitemap(map)
+        else
+          filename = (opts.output ? opts.output.to_s : url.hostname.to_s + ".json")
+          sitemap = gen_json_sitemap(map)
+        end
+
+        File.write(File.expand_path(filename, __DIR__), sitemap.to_s, mode: "w+")
+        puts "Wrote sitemap to #{filename}"
+      end
+
+      def gen_xml_sitemap(map)
+        XML.build(indent: "    ", encoding: "UTF-8") do |xml|
+          xml.element("urlset", xmlns: "http://www.sitemaps.org/schemas/sitemap/0.9") do
+            map[:pages].each do |page|
+              xml.element("url") do
+                lastmod = page[:lastmod]
+
+                xml.element("loc") { xml.text page[:url] }
+                xml.element("lastmod") { xml.text "#{lastmod[:year]}-#{lastmod[:month]}-#{lastmod[:day]}" }
+                xml.element("changefreq") { xml.text page[:changefreq] }
+                xml.element("priority") { xml.text page[:priority] }
+              end
+            end
+          end
+        end
+      end
+
+      def gen_json_sitemap(map)
+        map.to_pretty_json
+      end
+    end
+  end
+end