Permalink
Switch branches/tags
Nothing to show
Find file Copy path
Fetching contributors…
Cannot retrieve contributors at this time
executable file 146 lines (127 sloc) 4.53 KB
#!/usr/bin/env ruby
# frozen_string_literal: true
require "base64"
require "fileutils"
require "json"
require "open-uri"
require "optparse"
require "pathname"
require "shellwords"
require_relative "readme"
require_relative "utils"
CACHE_DIR = ".cache"
REPO_JSON_PATH = "_data/repos.json"
THUMBNAIL_CACHE = File.join(CACHE_DIR, "thumbnails")
THUMBNAIL_IMG_DIR = "static/img/thumbnails"
# TODO: replace this by path relative to README URL
RAW_URL_TPL = "https://raw.githubusercontent.com/${nameWithOwner}/master/"
GITHUB_DOUBLE_RAW_URI_RE = %r{
(https://raw.githubusercontent.com/[^/]+/[^/]+/)raw/(master/.+)(\?[^/]*)$
}x
options = {}
OptionParser.new do |opts|
opts.on("--from PATTERN", "Update repos matching pattern and beyond") do |p|
options[:from_pattern] = p.include?("/") ? p : "*/#{p}"
end
opts.on("-P", "--pattern PATTERN", "Update repos matching pattern") do |p|
options[:pattern] = p.include?("/") ? p : "*/#{p}"
end
opts.on("-f", "--force", "Force updates") do |f|
options[:force] = f
end
opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
options[:verbose] = v
end
end.parse!
verbose = options[:verbose]
repos = read_json(REPO_JSON_PATH)
# Update nwo's thumbnail path. Write the file immediately, since this doesn't
# add much time and makes the next run faster if we're stopped in the middle.
def update_thumbnail_path!(repos, nwo, thumbnail_path)
relpath = thumbnail_path && Pathname.new(thumbnail_path).relative_path_from(
Pathname.new(THUMBNAIL_CACHE)
).to_s
repo = repos.find { |r| r["nameWithOwner"] == nwo }
return if repo["thumbnailPath"] == relpath
repo["thumbnailPath"] = relpath
write_json! REPO_JSON_PATH, repos
end
# Returns local relative path
# rubocop:disable Metrics/AbcSize
# rubocop:disable Metrics/CyclomaticComplexity
# rubocop:disable Metrics/MethodLength
# rubocop:disable Metrics/PerceivedComplexity
def update_thumbnail!(repo, options)
nwo = repo["nameWithOwner"]
verbose = options[:verbose]
raw_url_prefix = RAW_URL_TPL.sub("${nameWithOwner}", nwo)
metadata_path = File.join(THUMBNAIL_CACHE, nwo, "metadata.json")
FileUtils.mkdir_p File.dirname(metadata_path)
metadata = File.exist?(metadata_path) ? read_json(metadata_path) : {}
thumbnail_path = metadata["thumbnail_path"]
if metadata.include?("thumbnail_path") && !options[:force]
if !thumbnail_path
puts "#{nwo}: no image (cached)" if verbose
return nil
else
puts "#{nwo}: #{thumbnail_path} (cached)" if verbose
return File.join(THUMBNAIL_CACHE, nwo, thumbnail_path)
end
end
print "#{nwo}: "
STDOUT.flush
readme = Readme.from_github_nwo(nwo)
metadata[:readme_name] = readme&.name
thumbnail_url = readme&.thumbnail_url
unless thumbnail_url
puts(readme ? "#{readme.name} doesn't contain a thumbnail" : "no README")
Dir.glob(File.join(THUMBNAIL_CACHE, nwo, "thumbnail.*")).each do |path|
FileUtils.rm path
end
metadata["thumbnail_path"] = nil
return nil
end
metadata["thumbnail_url"] = thumbnail_url
thumbnail_url = URI.join(raw_url_prefix, thumbnail_url).to_s
if thumbnail_url =~ GITHUB_DOUBLE_RAW_URI_RE
thumbnail_url = "#{$1}#{$2}"
end
thumbnail_ext = File.extname(thumbnail_url).sub(/\?.*/, "")
thumbnail_path = File.join(THUMBNAIL_CACHE, nwo, "thumbnail#{thumbnail_ext}")
puts "#{thumbnail_url} -> #{thumbnail_path}"
FileUtils.mkdir_p File.dirname(thumbnail_path)
# TODO: use etag to skip download
URI.parse(thumbnail_url).open do |f|
IO.copy_stream(f, thumbnail_path)
end
metadata["thumbnail_path"] = File.basename(thumbnail_path)
return thumbnail_path
ensure
write_json! metadata_path, metadata if metadata
end
# rubocop:enable Metrics/AbcSize
# rubocop:enable Metrics/CyclomaticComplexity
# rubocop:enable Metrics/MethodLength
# rubocop:enable Metrics/PerceivedComplexity
repos.sort_by { |r| r["nameWithOwner"] }.each do |repo|
nwo = repo["nameWithOwner"]
next if options[:pattern] && !File.fnmatch(options[:pattern], nwo)
next if options[:from_pattern] && !File.fnmatch(options[:from_pattern], nwo)
options[:from_pattern] = nil
path = update_thumbnail!(repo, options)
update_thumbnail_path! repos, nwo, path
end
puts "\nCopy #{THUMBNAIL_CACHE} -> #{THUMBNAIL_IMG_DIR}" if verbose
FileUtils.mkdir_p File.dirname(THUMBNAIL_IMG_DIR)
system <<~"SHELL"
rsync \
#{Shellwords.escape THUMBNAIL_CACHE}/ \
#{Shellwords.escape THUMBNAIL_IMG_DIR} \
#{verbose ? '--verbose' : '--quiet'} \
--checksum \
--delete \
--exclude metadata.json \
--exclude .DS_Store \
--prune-empty-dirs \
--recursive
SHELL