make-snapshot

#!/usr/bin/ruby -s
# -*- coding: us-ascii -*-
require 'rubygems'
require 'rubygems/package'
require 'rubygems/package/tar_writer'
require 'uri'
require 'digest/sha1'
require 'digest/sha2'
require 'fileutils'
require 'shellwords'
require 'tmpdir'
require 'pathname'
require 'yaml'
require 'json'
require File.expand_path("../lib/vcs", __FILE__)
require File.expand_path("../lib/colorize", __FILE__)
STDOUT.sync = true

$srcdir ||= nil
$archname = nil if ($archname ||= nil) == ""
$keep_temp ||= nil
$patch_file ||= nil
$packages ||= nil
$digests ||= nil
$no7z ||= nil
$tooldir = File.expand_path("..", __FILE__)
$unicode_version = nil if ($unicode_version ||= nil) == ""
$colorize = Colorize.new

def usage
  <<USAGE
usage: #{File.basename $0} [option...] new-directory-to-save [version ...]
options:
  -srcdir=PATH          source directory path
  -archname=NAME        make the basename of snapshots NAME
  -keep_temp            keep temporary working directory
  -patch_file=PATCH     apply PATCH file after export
  -packages=PKG[,...]   make PKG packages (#{PACKAGES.keys.join(", ")})
  -digests=ALG[,...]    show ALG digests (#{DIGESTS.join(", ")})
  -unicode_version=VER  Unicode version to generate encodings
  -help, --help         show this message
version:
  master, trunk, stable, branches/*, tags/*, X.Y, X.Y.Z, X.Y.Z-pL
each versions may be followed by optional @revision.
USAGE
end

DIGESTS = %w[SHA1 SHA256 SHA512]
PACKAGES = {
  "tar"  => %w".tar",
  "bzip" => %w".tar.bz2 bzip2 -c",
  "gzip" => %w".tar.gz  gzip -c",
  "xz"   => %w".tar.xz  xz -c",
  "zip"  => %w".zip     zip -Xqr",
}
DEFAULT_PACKAGES = PACKAGES.keys - ["tar"]
if !$no7z and system("7z", out: IO::NULL)
  PACKAGES["gzip"] = %w".tar.gz 7z a dummy -tgzip -mx -so"
  PACKAGES["zip"]  = %w".zip    7z a -tzip -mx -mtc=off" << {out: IO::NULL}
elsif gzip = ENV.delete("GZIP")
  PACKAGES["gzip"].concat(gzip.shellsplit)
end

if mflags = ENV["GNUMAKEFLAGS"] and /\A-(\S*)j\d*/ =~ mflags
  mflags = mflags.gsub(/(\A|\s)(-\S*)j\d*/, '\1\2')
  mflags.strip!
  ENV["GNUMAKEFLAGS"] = (mflags unless mflags.empty?)
end
ENV["LC_ALL"] = ENV["LANG"] = "C"
# https git clone is disabled at git.ruby-lang.org/ruby.git.
GITURL = URI.parse("https://github.com/ruby/ruby.git")
RUBY_VERSION_PATTERN = /^\#define\s+RUBY_VERSION\s+"([\d.]+)"/

ENV["TZ"] = "UTC"

class String
  # for older ruby
  alias bytesize size unless method_defined?(:bytesize)
end

class Dir
  def self.mktmpdir(path)
    path = File.join(tmpdir, path+"-#{$$}-#{rand(100000)}")
    begin
      mkdir(path)
    rescue Errno::EEXIST
      path.succ!
      retry
    end
    path
  end unless respond_to?(:mktmpdir)
end

$packages &&= $packages.split(/[, ]+/).tap {|pkg|
  if all = pkg.index("all")
    pkg[all, 1] = DEFAULT_PACKAGES - pkg
  end
  pkg -= PACKAGES.keys
  pkg.empty? or abort "#{File.basename $0}: unknown packages - #{pkg.join(", ")}"
}
$packages ||= DEFAULT_PACKAGES

$digests &&= $digests.split(/[, ]+/).tap {|dig|
  dig -= DIGESTS
  dig.empty? or abort "#{File.basename $0}: unknown digests - #{dig.join(", ")}"
}
$digests ||= DIGESTS

$patch_file &&= File.expand_path($patch_file)
PATH = ENV["PATH"].split(File::PATH_SEPARATOR)
def PATH.executable_env(var, command = nil)
  command = ENV[var] ||= (command or return)
  cmd, = command.shellsplit
  unless any? {|dir|
      file = File.expand_path(cmd, dir)
      File.file?(file) and File.executable?(file)
    }
    abort "#{File.basename $0}: #{var} command not found - #{cmd}"
  end
  command
end

PATH.executable_env("MV", "mv")
PATH.executable_env("RM", "rm -f")
PATH.executable_env("AUTOCONF", "autoconf")

%w[BASERUBY RUBY MINIRUBY].each do |var|
  cmd = PATH.executable_env(var, "ruby")
  help = IO.popen("#{cmd} --help", err: %i[child out], &:read)
  unless $?.success? and /ruby/ =~ help
    abort "#{File.basename $0}: #{var} ruby not found - #{cmd}"
  end
  IO.popen("#{cmd} --disable-gem -eexit", err: %i[child out], &:read)
  cmd += ' --disable-gem' if $?.success?
  ENV[var] = cmd
end

if defined?($help) or defined?($_help)
  puts usage
  exit
end
unless destdir = ARGV.shift
  abort usage
end
revisions = ARGV.empty? ? [nil] : ARGV

if defined?($exported)
  abort "#{File.basename $0}: -exported option is deprecated; use -srcdir instead"
end

FileUtils.mkpath(destdir)
destdir = File.expand_path(destdir)
tmp = Dir.mktmpdir("ruby-snapshot")
FileUtils.mkpath(tmp)
at_exit {
  Dir.chdir "/"
  FileUtils.rm_rf(tmp)
} unless $keep_temp

def tar_create(tarball, dir)
  header = Gem::Package::TarHeader
  dir_type = "5"
  uname = gname = "ruby"
  File.open(tarball, "wb") do |f|
    w = Gem::Package::TarWriter.new(f)
    list = Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH)
    list.reject! {|name| name.end_with?("/.")}
    list.sort_by! {|name| name.split("/")}
    list.each do |path|
      next if File.basename(path) == "."
      s = File.stat(path)
      mode = 0644
      case
      when s.file?
        type = nil
        size = s.size
        mode |= 0111 if s.executable?
      when s.directory?
        path += "/"
        type = dir_type
        size = 0
        mode |= 0111
      else
        next
      end
      name, prefix = w.split_name(path)
      h = header.new(name: name, prefix: prefix, typeflag: type,
                     mode: mode, size: size, mtime: s.mtime,
                     uname: uname, gname: gname)
      f.write(h)
      if size > 0
        IO.copy_stream(path, f)
        f.write("\0" * (-size % 512))
      end
    end
  end
  true
rescue => e
  warn e.message
  false
end

def touch_all(time, pattern, opt, &cond)
  Dir.glob(pattern, opt) do |n|
    stat = File.stat(n)
    if stat.file? or stat.directory?
      next if cond and !yield(n, stat)
      File.utime(time, time, n)
    end
  end
rescue
  false
else
  true
end

class MAKE < Struct.new(:prog, :args)
  def initialize(vars)
    vars = vars.map {|arg| arg.join("=")}
    super(ENV["MAKE"] || ENV["make"] || "make", vars)
  end

  def run(target)
    err = IO.pipe do |r, w|
      begin
        pid = Process.spawn(self.prog, *self.args, target, {:err => w, r => :close})
        w.close
        r.read
      ensure
        Process.wait(pid)
      end
    end
    if $?.success?
      true
    else
      STDERR.puts err
      $colorize.fail("#{target} failed")
      false
    end
  end
end

def measure
  clock = Process::CLOCK_MONOTONIC
  t0 = Process.clock_gettime(clock)
  STDOUT.flush
  result = yield
  printf(" %6.3f", Process.clock_gettime(clock) - t0)
  STDOUT.flush
  result
end

def package(vcs, rev, destdir, tmp = nil)
  pwd = Dir.pwd
  patchlevel = false
  prerelease = false
  if rev and revision = rev[/@(\h+)\z/, 1]
    rev = $`
  end
  case rev
  when nil
    url = nil
  when /\A(?:master|trunk)\z/
    url = vcs.trunk
  when /\Abranches\//
    url = vcs.branch($')
  when /\Atags\//
    url = vcs.tag($')
  when /\Astable\z/
    vcs.branch_list("ruby_[0-9]*") {|n| url = n[/\Aruby_\d+_\d+\z/]}
    url &&= vcs.branch(url)
  when /\A(.*)\.(.*)\.(.*)-(preview|rc)(\d+)/
    prerelease = true
    tag = "#{$4}#{$5}"
    url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}#{$5}")
  when /\A(.*)\.(.*)\.(.*)-p(\d+)/
    patchlevel = true
    tag = "p#{$4}"
    url = vcs.tag("v#{$1}_#{$2}_#{$3}_#{$4}")
  when /\A(\d+)\.(\d+)(?:\.(\d+))?\z/
    if $3 && ($1 > "2" || $1 == "2" && $2 >= "1")
      patchlevel = true
      tag = ""
      url = vcs.tag("v#{$1}_#{$2}_#{$3}")
    else
      url = vcs.branch("ruby_#{rev.tr('.', '_')}")
    end
  else
    warn "#{$0}: unknown version - #{rev}"
    return
  end
  if info = vcs.get_revisions(url)
    modified = info[2]
  else
    _, _, modified = VCS::Null.new(nil).get_revisions(url)
  end
  if !revision and info
    revision = info
    url ||= vcs.branch(revision[3])
    revision = revision[1]
  end
  version = nil
  unless revision
    url = vcs.trunk
    vcs.grep(RUBY_VERSION_PATTERN, url, "version.h") {version = $1}
    unless rev == version
      warn "#{$0}: #{rev} not found"
      return
    end
    revision = vcs.get_revisions(url)[1]
  end

  v = "ruby"
  puts "Exporting #{rev}@#{revision}"
  exported = tmp ? File.join(tmp, v) : v
  unless vcs.export(revision, url, exported, true) {|line| print line}
    warn("Export failed")
    return
  end
  if $srcdir
    Dir.glob($srcdir + "/{tool/config.{guess,sub},gems/*.gem,.downloaded-cache/*,enc/unicode/data/**/*.txt}") do |file|
      puts "copying #{file}" if $VERBOSE
      dest = exported + file[$srcdir.size..-1]
      FileUtils.mkpath(File.dirname(dest))
      begin
        FileUtils.cp_r(file, dest)
        FileUtils.chmod_R("a+rwX,go-w", dest)
      rescue SystemCallError
      end
    end
  end

  Dir.glob("#{exported}/.*.yml") do |file|
    FileUtils.rm(file, verbose: $VERBOSE)
  end

  status = IO.read(File.dirname(__FILE__) + "/prereq.status")
  Dir.chdir(tmp) if tmp

  if !File.directory?(v)
    v = Dir.glob("ruby-*").select(&File.method(:directory?))
    v.size == 1 or abort "#{File.basename $0}: not exported"
    v = v[0]
  end

  File.open("#{v}/revision.h", "wb") {|f|
    f.puts vcs.revision_header(revision, modified)
  }
  version ||= (versionhdr = IO.read("#{v}/version.h"))[RUBY_VERSION_PATTERN, 1]
  version ||=
    begin
      include_ruby_versionhdr = IO.read("#{v}/include/ruby/version.h")
      api_major_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MAJOR\s+([\d.]+)/, 1]
      api_minor_version = include_ruby_versionhdr[/^\#define\s+RUBY_API_VERSION_MINOR\s+([\d.]+)/, 1]
      version_teeny = versionhdr[/^\#define\s+RUBY_VERSION_TEENY\s+(\d+)/, 1]
      [api_major_version, api_minor_version, version_teeny].join('.')
    end
  version or return
  if patchlevel
    unless tag.empty?
      versionhdr ||= IO.read("#{v}/version.h")
      patchlevel = versionhdr[/^\#define\s+RUBY_PATCHLEVEL\s+(\d+)/, 1]
      tag = (patchlevel ? "p#{patchlevel}" : vcs.revision_name(revision))
    end
  elsif prerelease
    versionhdr ||= IO.read("#{v}/version.h")
    versionhdr.sub!(/^\#\s*define\s+RUBY_PATCHLEVEL_STR\s+"\K.+?(?=")/, tag) or raise "no match of RUBY_PATCHLEVEL_STR to replace"
    IO.write("#{v}/version.h", versionhdr)
  else
    tag ||= vcs.revision_name(revision)
  end

  if $archname
    n = $archname
  elsif tag.empty?
    n = "ruby-#{version}"
  else
    n = "ruby-#{version}-#{tag}"
  end
  File.directory?(n) or File.rename v, n
  v = n

  if $patch_file && !system(*%W"patch -d #{v} -p0 -i #{$patch_file}")
    puts $colorize.fail("patching failed")
    return
  end

  class << (clean = [])
    def add(n) push(n)
      n
    end
    def create(file, content = "", &block)
      add(file)
      if block
        File.open(file, "wb", &block)
      else
        File.binwrite(file, content)
      end
    end
  end

  Dir.chdir(v) do
    unless File.exist?("ChangeLog")
      vcs.export_changelog(url, nil, revision, "ChangeLog")
    end

    unless touch_all(modified, "**/*", File::FNM_DOTMATCH)
      modified = nil
      colors = %w[red yellow green cyan blue magenta]
      "take a breath, and go ahead".scan(/./) do |c|
        if c == ' '
          print c
        else
          colors.push(color = colors.shift)
          print $colorize.decorate(c, color)
        end
        sleep(c == "," ? 0.7 : 0.05)
      end
      puts
    end

    clean.create("cross.rb") do |f|
      f.puts "Object.__send__(:remove_const, :CROSS_COMPILING) if defined?(CROSS_COMPILING)"
      f.puts "CROSS_COMPILING=true"
      f.puts "Object.__send__(:remove_const, :RUBY_PLATFORM)"
      f.puts "RUBY_PLATFORM='none'"
      f.puts "Object.__send__(:remove_const, :RUBY_VERSION)"
      f.puts "RUBY_VERSION='#{version}'"
    end
    puts "cross.rb:", File.read("cross.rb").gsub(/^/, "> "), "" if $VERBOSE
    unless File.exist?("configure")
      print "creating configure..."
      unless system([ENV["AUTOCONF"]]*2)
        puts $colorize.fail(" failed")
        return
      end
      puts $colorize.pass(" done")
    end
    clean.add("autom4te.cache")
    clean.add("enc/unicode/data")
    print "creating prerequisites..."
    if File.file?("common.mk") && /^prereq/ =~ commonmk = IO.read("common.mk")
      puts
      extout = clean.add('tmp')
      begin
        status = IO.read("tool/prereq.status")
      rescue Errno::ENOENT
        # use fallback file
      end
      clean.create("config.status", status)
      clean.create("noarch-fake.rb", "require_relative 'cross'\n")
      FileUtils.mkpath(hdrdir = "#{extout}/include/ruby")
      File.binwrite("#{hdrdir}/config.h", "")
      FileUtils.mkpath(defaults = "#{extout}/rubygems/defaults")
      File.binwrite("#{defaults}/operating_system.rb", "")
      File.binwrite("#{defaults}/ruby.rb", "")
      miniruby = ENV['MINIRUBY'] + " -I. -I#{extout} -rcross"
      baseruby = ENV["BASERUBY"]
      mk = (IO.read("template/Makefile.in") rescue IO.read("Makefile.in")).
             gsub(/^@.*\n/, '')
      vars = {
        "EXTOUT"=>extout,
        "PATH_SEPARATOR"=>File::PATH_SEPARATOR,
        "MINIRUBY"=>miniruby,
        "RUBY"=>ENV["RUBY"],
        "BASERUBY"=>baseruby,
        "PWD"=>Dir.pwd,
        "ruby_version"=>version,
        "MAJOR"=>api_major_version,
        "MINOR"=>api_minor_version,
        "TEENY"=>version_teeny,
        "VPATH"=>(ENV["VPATH"] || "include/ruby"),
        "PROGRAM"=>(ENV["PROGRAM"] || "ruby"),
        "BUILTIN_TRANSOBJS"=>(ENV["BUILTIN_TRANSOBJS"] || "newline.o"),
      }
      status.scan(/^s([%,])@([A-Za-z_][A-Za-z_0-9]*)@\1(.*?)\1g$/) do
        vars[$2] ||= $3
      end
      vars.delete("UNICODE_FILES") # for stable branches
      vars["UNICODE_VERSION"] = $unicode_version if $unicode_version
      args = vars.dup
      mk.gsub!(/@([A-Za-z_]\w*)@/) {args.delete($1); vars[$1] || ENV[$1]}
      mk << commonmk.gsub(/\{\$([^(){}]*)[^{}]*\}/, "").sub(/^revision\.tmp::$/, '\& Makefile')
      mk << <<-'APPEND'

update-download:: touch-unicode-files
prepare-package: prereq after-update
clean-cache: $(CLEAN_CACHE)
after-update:: extract-gems
extract-gems: update-gems
update-gems:
$(UNICODE_SRC_DATA_DIR)/.unicode-tables.time:
touch-unicode-files:
      APPEND
      clean.create("Makefile", mk)
      clean.create("revision.tmp")
      clean.create(".revision.time")
      ENV["CACHE_SAVE"] = "no"
      make = MAKE.new(args)
      return unless make.run("update-download")
      clean.push("rbconfig.rb", ".rbconfig.time", "enc.mk", "ext/ripper/y.output", ".revision.time")
      Dir.glob("**/*") do |dest|
        next unless File.symlink?(dest)
        orig = File.expand_path(File.readlink(dest), File.dirname(dest))
        File.unlink(dest)
        FileUtils.cp_r(orig, dest)
      end
      File.utime(modified, modified, *Dir.glob(["tool/config.{guess,sub}", "gems/*.gem", "tool"]))
      return unless make.run("prepare-package")
      return unless make.run("clean-cache")
      if modified
        new_time = modified + 2
        touch_all(new_time, "**/*", File::FNM_DOTMATCH) do |name, stat|
          stat.mtime > modified unless clean.include?(name)
        end
        modified = new_time
      end
      print "prerequisites"
    else
      system(*%W[#{PATH.executable_env("YACC", "bison")} -o parse.c parse.y])
    end
    vcs.after_export(".") if exported
    clean.concat(Dir.glob("ext/**/autom4te.cache"))
    clean.add(".downloaded-cache")
    if File.exist?("gems/bundled_gems")
      gems = Dir.glob("gems/*.gem")
      gems -= File.readlines("gems/bundled_gems").map {|line|
        next if /^\s*(?:#|$)/ =~ line
        name, version, _ = line.split(' ')
        "gems/#{name}-#{version}.gem"
      }
      clean.concat(gems)
    else
      clean.add("gems")
    end
    FileUtils.rm_rf(clean)
    if modified
      touch_all(modified, "**/*/", 0) do |name, stat|
        stat.mtime > modified
      end
      File.utime(modified, modified, ".")
    end
    unless $?.success?
      puts $colorize.fail(" failed")
      return
    end
    puts $colorize.pass(" done")
  end

  if v == "."
    v = File.basename(Dir.pwd)
    Dir.chdir ".."
  else
    Dir.chdir(File.dirname(v))
    v = File.basename(v)
  end

  tarball = nil
  return $packages.collect do |mesg|
    (ext, *cmd) = PACKAGES[mesg]
    File.directory?(destdir) or FileUtils.mkpath(destdir)
    file = File.join(destdir, "#{$archname||v}#{ext}")
    case ext
    when /\.tar/
      if tarball
        next if tarball.empty?
      else
        tarball = ext == ".tar" ? file : "#{$archname||v}.tar"
        print "creating tarball... #{tarball}"
        if measure {tar_create(tarball, v)}
          puts $colorize.pass(" done")
          File.utime(modified, modified, tarball) if modified
          next if tarball == file
        else
          puts $colorize.fail(" failed")
          tarball = ""
          next
        end
      end
      print "creating #{mesg} tarball... #{file}"
      done = measure {system(*cmd, tarball, out: file)}
    else
      print "creating #{mesg} archive... #{file}"
      if Hash === cmd.last
        *cmd, opt = *cmd
        cmd << file << v << opt
      else
        (cmd = cmd.dup) << file << v
      end
      done = measure {system(*cmd)}
    end
    if done
      puts $colorize.pass(" done")
      file
    else
      puts $colorize.fail(" failed")
      nil
    end
  end.compact
ensure
  FileUtils.rm_rf(tmp ? File.join(tmp, v) : v) if v and !$keep_temp
  Dir.chdir(pwd)
end

if [$srcdir, ($git||=nil)].compact.size > 1
  abort "#{File.basename $0}: -srcdir and -git are exclusive"
end
if $srcdir
  vcs = VCS.detect($srcdir)
elsif $git
  abort "#{File.basename $0}: use -srcdir with cloned local repository"
else
  begin
    vcs = VCS.detect(File.expand_path("../..", __FILE__))
  rescue VCS::NotFoundError
    abort "#{File.expand_path("../..", __FILE__)}: cannot find git repository"
  end
end

release_date = Time.now.getutc
info = {}

success = true
revisions.collect {|rev| package(vcs, rev, destdir, tmp)}.flatten.each do |name|
  if !name
    success = false
    next
  end
  str = File.binread(name)
  pathname = Pathname(name)
  basename = pathname.basename.to_s
  extname = pathname.extname.sub(/\A\./, '')
  version = basename[/\Aruby-(.*)\.(?:tar|zip)/, 1]
  key = basename[/\A(.*)\.(?:tar|zip)/, 1]
  info[key] ||= Hash.new{|h,k|h[k]={}}
  info[key]['version'] = version if version
  info[key]['date'] = release_date.strftime('%Y-%m-%d')
  if version
    info[key]['post'] = "/en/news/#{release_date.strftime('%Y/%m/%d')}/ruby-#{version.tr('.', '-')}-released/"
    info[key]['url'][extname] = "https://cache.ruby-lang.org/pub/ruby/#{version[/\A\d+\.\d+/]}/#{basename}"
  else
    info[key]['filename'][extname] = basename
  end
  info[key]['size'][extname] = str.bytesize
  puts "* #{$colorize.pass(name)}"
  puts "      SIZE:   #{str.bytesize} bytes"
  $digests.each do |alg|
    digest = Digest(alg).hexdigest(str)
    info[key][alg.downcase][extname] = digest
    printf "      %-8s%s\n", "#{alg}:", digest
  end
end

yaml = info.values.to_yaml
json = info.values.to_json
puts "#{$colorize.pass('YAML:')}"
puts yaml
puts "#{$colorize.pass('JSON:')}"
puts json
infodir = Pathname(destdir) + 'info'
infodir.mkpath
(infodir+'info.yml').write(yaml)
(infodir+'info.json').write(json)

exit false if !success

# vim:fileencoding=US-ASCII sw=2 ts=4 noexpandtab ff=unix