diff --git a/_plugins/llms.rb b/_plugins/llms.rb new file mode 100644 index 0000000000..fec7c0ab0d --- /dev/null +++ b/_plugins/llms.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +# Generates /llms.txt and /llms-full.txt (llmstxt.org) from the site +# itself, so neither can drift from the published pages. llms.txt lists +# every English page grouped by top-level section, with the title from +# the front matter and the first sentence of the page as description. +# llms-full.txt inlines the full Markdown of the same pages for tools +# that want the whole site in one fetch. News posts appear only as the +# ten latest links, because two decades of announcements would add over +# 1 MB of archival text. Rendering is shared with the .md alternates so +# the outputs stay identical to what those routes serve. +module LlmsTxt + SITE_NAME = "Ruby Programming Language" + TAGLINE = "Ruby is a dynamic, open source programming language " \ + "with a focus on simplicity and productivity. It has an " \ + "elegant syntax that is natural to read and easy to write." + + # Pages without a title (layout: null fragments such as the homepage + # code examples) are not standalone documents and are left out. + def self.pages(site) + site.pages.select do |page| + page.url.start_with?("/en/") && page.data["markdown_url"] && page.data["title"] + end.sort_by(&:url) + end + + def self.posts(site) + site.posts.docs.select {|post| post.data["lang"] == "en" }.last(10).reverse + end + + def self.header(site) + stable = site.data["downloads"]["stable"].first + "# #{SITE_NAME}\n\n> #{TAGLINE} The current stable version is Ruby #{stable}.\n" + end + + # First sentence of the first prose paragraph, with images, links, + # HTML tags, emphasis and kramdown attribute lines stripped. + def self.description(markdown) + markdown.split(/\n\s*\n/).each do |block| + block = block.strip + next if block.empty? || block.start_with?("<", "#", ">", "{%", "|", "*", "-", "!") + + text = block.lines.map(&:strip).reject {|line| line.start_with?("{:") }.join(" ") + text = text.gsub(/!\[[^\]]*\](\([^)]*\)|\[[^\]]*\])?/, "") + .gsub(/\[([^\]]*)\](\([^)]*\)|\[[^\]]*\])/, '\1') + .gsub(/<[^>]+>/, " ") + .gsub(/[*_`]/, "") + .squeeze(" ").strip + next if text.empty? + + return text[/\A.{10,}?[.!?](?=\s|\z)/m] || text[0, 200] + end + nil + end + + def self.index(site, rendered) + out = +header(site) + + pages(site).group_by {|page| page.url.split("/")[2] }.each do |slug, section| + front = section.find {|page| page.url == "/en/#{slug}/" } + out << "\n## #{front ? front.data["title"] : slug.capitalize}\n\n" + section.each do |page| + desc = description(rendered[page]) + out << "- [#{page.data["title"]}](#{site.config["url"]}#{page.data["markdown_url"]})" + out << ": #{desc}" if desc + out << "\n" + end + end + + out << "\n## Recent News\n\n" + posts(site).each do |post| + url = site.config["url"] + post.data["markdown_url"] + out << "- [#{post.data["title"]}](#{url}): #{post.date.strftime("%Y-%m-%d")}\n" + end + + out << "\n## Optional\n\n" + out << "- [llms-full.txt](#{site.config["url"]}/llms-full.txt): " \ + "All English pages of this site concatenated into a single Markdown file\n" + end + + def self.full(site, rendered) + sections = pages(site).map do |page| + "# #{page.data["title"]}\n\n" \ + "Source: #{site.config["url"]}#{page.data["markdown_url"]}\n\n" \ + "#{rendered[page]}\n" + end + + ([header(site)] + sections).join("\n---\n\n") + end +end + +Jekyll::Hooks.register :site, :post_write do |site| + rendered = LlmsTxt.pages(site).to_h do |page| + [page, MarkdownAlternates.markdown_for(site, page).strip] + end + + File.write(site.in_dest_dir("llms.txt"), LlmsTxt.index(site, rendered)) + File.write(site.in_dest_dir("llms-full.txt"), LlmsTxt.full(site, rendered)) +end diff --git a/lib/linter.rb b/lib/linter.rb index 4296e39c41..925b75ab9a 100644 --- a/lib/linter.rb +++ b/lib/linter.rb @@ -18,6 +18,7 @@ class Linter %r{\Aadmin/index\.md}, %r{\A[^/]*/examples/}, %r{\A_includes/}, + %r{\A_site/}, %r{\Atest/}, %r{\Anode_modules/} ].freeze