Permalink
Fetching contributors…
Cannot retrieve contributors at this time
executable file 884 lines (725 sloc) 26.7 KB
#!/usr/bin/env ruby
# Copyright (C) 2014 Kano Computing Ltd.
# License: http://www.gnu.org/licenses/gpl-2.0.txt GNU General Public License v2
require "thor"
require "fileutils"
require "io/console"
require "dr"
require "dr/repo"
require "dr/gitpackage"
require "dr/debpackage"
require "dr/buildroot"
require "dr/pkgversion"
require "dr/shellcmd"
require "dr/logger"
require "dr/config"
require "dr/server"
require "dr/threadpool"
class ExtendedThor < Thor
private
include Dr::Logger
def initialize(*args)
super
Dr::Logger::set_verbosity options[:verbosity]
if not options[:log_file].nil?
file = File.open options[:log_file],"w"
Dr::Logger::set_logfile(file)
end
end
def get_repo_handle
if options.has_key? "repo"
if Dr.config.repositories.has_key? options["repo"]
Dr::Repo.new Dr.config.repositories[options["repo"]][:location]
else
Dr::Repo.new options["repo"]
end
else
if Dr.config.default_repo != nil
Dr::Repo.new Dr.config.repositories[Dr.config.default_repo][:location]
else
log :warn, "No repo was specified, using '#{Dir.pwd}'."
Dr::Repo.new Dir.pwd
end
end
end
end
class Archive < ExtendedThor
#desc "save TAG", "make a snapshot of the current archive"
#def save(tag)
#end
#desc "restore TAG", "replace the current archive with an earlier snapshot"
#def restore(tag)
#end
#desc "list-versions", "show all snapshots"
#map "list-versions" => :list_versions
#def list_versions
#end
end
class Conf < ExtendedThor
desc "repo KEY [VALUE]", "Configuration of the whole repository"
def repo(key, value=nil)
repo = get_repo_handle
metadata = repo.get_configuration
if value == nil
value = dot_get_value metadata, key
puts value if value
else
repo.set_configuration dot_set_value metadata, key, value
end
end
desc "package PKG-NAME KEY [VALUE]", "Package-specific configuration options"
def package(pkg_name, key, value=nil)
repo = get_repo_handle
pkg = repo.get_package pkg_name
metadata = pkg.get_configuration
if value == nil
value = dot_get_value metadata, key
puts value if value
else
pkg.set_configuration dot_set_value metadata, key, value
end
end
private
def dot_set_value(dict, key, value)
levels = key.split(".").map {|l| l.to_sym}
raise "Incorrect key" if levels.length == 0
begin
last = levels.pop
object = dict
levels.each do |l|
object[l] = {} unless object.has_key? l
object = object[l]
end
if value.length > 0
object[last] = value
else
object.delete last
end
rescue
log :err, "The configuration key '#{key}' isn't right"
raise "Incorrect key"
end
dict
end
def dot_get_value(dict, key)
levels = key.split(".").map {|l| l.to_sym}
raise "Incorrect key" if levels.length == 0
begin
last = levels.pop
object = dict
levels.each do |l|
object = object[l]
end
return object[last]
rescue
log :err, "The configuration key '#{key}' isn't right"
raise "Incorrect key"
end
end
end
class List < ExtendedThor
desc "packages", "Show a list of source packages in the repo"
def packages()
repo = get_repo_handle
log :info, "Listing all source packages in the repository"
repo.list_packages.each do |pkg|
log :info, " #{pkg.name.fg "orange"}"
end
end
desc "gitrepos SUITE", "Show a list of git repo names for source packages"
def gitrepos(suite)
repo = get_repo_handle
suites = repo.get_suites
exists = suites.inject(false) { |r, s| r || s.include?(suite) }
raise "Suite '#{suite}' doesn't exist" unless exists
log :info, "Listing all git repos used by #{suite.fg "blue"}"
suite = repo.codename_to_suite suite
suite_packages = repo.list_packages
suite_packages.each do |pkg|
if pkg.is_a? Dr::GitPackage
versions = repo.get_subpackage_versions pkg.name
reponame = pkg.get_repo_name
if not versions[suite].empty?
log :info, " #{reponame.fg "orange"}"
end
end
end
end
desc "versions PKG-NAME", "DEPRECATED, please use builds instead"
def versions(pkg_name)
log :warn, "This subcommand is deprecated, please use builds instead"
builds pkg_name
end
desc "builds PKG-NAME", "Show the history of all builds of a package"
def builds(pkg_name)
repo = get_repo_handle
log :info, "Listing all builds of #{pkg_name.style "pkg-name"}"
suites = repo.get_suites
pkg = repo.get_package pkg_name
pkg.history.each do |version|
line = "#{version.style "version"}"
if pkg.build_exists? version
debs = repo.get_build pkg.name, version
metadata = repo.get_build_metadata pkg.name, version
if metadata.has_key? "branch"
open = "{".fg "dark-grey"
close = "}".fg "dark-grey"
line << " " + open + metadata["branch"].fg("blue") + close
end
subpkgs = debs.map { |p| File.basename(p).split("_")[0] }
end
open = "[".fg "dark-grey"
close = "]".fg "dark-grey"
if subpkgs.length == 0
line << " " + open + "broken".fg("red") + close
else
suites.each do |suite, codename|
codename = suite if codename == nil
colour = suite_to_colour suite
all_included = true
subpkgs.each do |subpkg|
unless repo.query_for_deb_version(suite, subpkg) == version
all_included = false
end
end
if all_included
if colour
line << " " + open + codename.fg(colour) + close
else
line << " " + open + codename + close
end
end
end
end
log :info, " #{line}"
end
end
desc "suite SUITE", "Show the names and versions of packages in the suite"
def suite(suite)
repo = get_repo_handle
suites = repo.get_suites
exists = suites.inject(false) { |r, s| r || s.include?(suite) }
raise "Suite '#{suite}' doesn't exist" unless exists
log :info, "Listing all the packages in #{suite.fg "blue"}"
suite = repo.codename_to_suite suite
suite_packages = repo.list_packages
suite_packages.each do |pkg|
versions = repo.get_subpackage_versions pkg.name
unless versions[suite].empty?
if versions[suite].length == 1 && versions[suite].has_key?(pkg.name)
log :info, " #{pkg.name.style "pkg-name"} " +
"#{versions[suite][pkg.name].style "version"}"
else
log :info, " #{pkg.name.style "pkg-name"}"
versions[suite].each do |subpkg, version|
log :info, " #{subpkg.style "subpkg-name"} " +
"#{version.style "version"}"
end
end
end
end
end
desc "codenames", "Show the codenames of the configured suites"
def codenames
repo = get_repo_handle
suites = repo.get_suites
suites.each do |suite, codename|
codename = suite if codename == nil
colour = suite_to_colour suite
log :info, "#{codename.fg colour}: #{suite}"
end
end
private
def suite_to_colour(suite)
colour = case suite
when "stable-security" then "magenta"
when "stable" then "red"
when "testing" then "yellow"
when "unstable" then "green"
else "cyan" end
return colour
end
end
class RepoCLI < ExtendedThor
class_option :repo, :type => :string, :aliases => "-r"
class_option :verbosity, :type => :string, :aliases => "-v", :default => "verbose"
class_option :log_file, :type => :string, :aliases => "-l", :default => nil
desc "init [LOCATION]", "setup a whole new repository from scratch"
def init(location=".")
log :info, "Initialising a debian repository at '#{location.fg("blue")}'"
repo_conf = {
:name => "Debian Repository",
:desc => "",
:arches => ["amd64"],
:components => ["main"],
:suites => ["stable-security", "stable", "testing", "unstable"],
:build_environment => :kano,
:codenames => []
}
name = ask " Repository name "<< "[#{repo_conf[:name].fg("yellow")}]:"
repo_conf[:name] = name if name.length > 0
desc = ask " Description [#{repo_conf[:desc]}]:"
repo_conf[:desc] = desc if desc.length > 0
puts " Default build environment [pick one]: "
Dr::config.build_environments.each do |id, benv|
puts " [#{id.to_s.fg "blue"}] #{benv[:name]}"
end
benv = nil
loop do
benv_str = ask " Your choice [#{repo_conf[:build_environment].to_s.fg "yellow"}]:"
benv = benv_str.to_sym
break if Dr::config.build_environments.has_key? benv_str.to_sym
end
repo_conf[:build_environment] = benv
# guess repo arches
repo_conf[:arches] = Dr::config.build_environments[benv][:arches]
loop do
str = ask " Architectures [#{repo_conf[:arches].join(" ").fg("yellow")}]:"
break if str.length == 0
# Determine the available architectures
avail = Dr.config.build_environments[benv][:arches]
arches = str.split(/\s+/)
arches_valid = arches.reduce(true) do |acc, arch|
if !avail.include?(arch)
puts " " + "#{arch.fg "yellow"}" +
" not supported by the build environments you selected"
acc = false
end
acc
end
next if !arches_valid
repo_conf[:arches] = arches
break
end
components = ask " Components [#{repo_conf[:components].join(" ").fg("yellow")}]:"
repo_conf[:components] = components.split(/\s+/) if components.length > 0
repo_conf[:gpg_name] = ""
while repo_conf[:gpg_name].length == 0
repo_conf[:gpg_name] = ask " Cert owner name (#{"required".fg("red")}):"
repo_conf[:gpg_name].strip!
end
repo_conf[:gpg_mail] = ""
while repo_conf[:gpg_mail].length == 0
repo_conf[:gpg_mail] = ask " Cert owner e-mail (#{"required".fg("red")}):"
repo_conf[:gpg_mail].strip!
end
print " Passphrase (#{"optional".fg("green")}): "
repo_conf[:gpg_pass] = STDIN.noecho(&:gets).chomp
print "\n"
repo_conf[:suites].each do |s|
codename = ask " Codename for '#{s.fg("yellow")}':"
repo_conf[:codenames].push codename
end
# TODO: Add CLI command to add suites
extra_suite_count = ask " Additional suites [#{'0'.fg("yellow")}]:"
extra_suite_count.to_i.times do |idx|
suite_name = nil
loop do
suite_name = ask " Suite #{idx + 1} name (#{"required".fg("red")}):"
if repo_conf[:suites].include? suite_name
log :warn, "Suite already exists, try again".fg("red")
next
end
break unless suite_name.empty?
end
repo_conf[:suites].push suite_name
codename = nil
loop do
codename = ask " Codename for suite #{idx + 1} (#{"required".fg("red")}):"
if repo_conf[:codenames].include? codename
log :warn, "Codename already exists, try again".fg("red")
next
end
break unless codename.empty?
end
repo_conf[:codenames].push codename
end
r = Dr::Repo.new location
r.setup repo_conf
end
desc "add", "introduce a new package to the build system"
method_option :git, :aliases => "-g",
:desc => "Add source package managed in a git repo"
method_option :deb, :aliases => "-d",
:desc => "Add a prebuilt binary deb package only"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "Proceed even if the package already exists"
method_option :branch, :aliases => "-b",
:desc => "Set a default branch other than master (valid only with --git)"
def add
repo = get_repo_handle
case
when options.has_key?("git")
branch = "master"
branch = options["branch"] if options.has_key? "branch"
Dr::GitPackage::setup repo, options["git"], branch
when options.has_key?("deb")
Dr::DebPackage::setup repo, options["deb"], options["force"]
else
raise ArgumentError, "Either --git or --deb must be specified"
end
end
desc "build PKG-NAME", "build a package from the sources"
method_option :branch, :aliases => "-b", :type => :string,
:desc => "build from a different branch"
method_option :push, :aliases => "-p", :type => :string,
:desc => "push to suite immediately after building"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "force build even when no changes have been made"
def build(pkg_name)
repo = get_repo_handle
force = false
force = options["force"] if options.has_key? "force"
branch = nil
branch = options["branch"] if options.has_key? "branch"
pkg = repo.get_package pkg_name
version = pkg.build branch, force
unless version
log :warn, "Build stopped (add -f to build anyway)"
return
end
if options["push"] && version
if options["push"] == "push"
repo.push pkg.name, version, "testing" # FIXME: should be configurable
else
if repo.codename_to_suite(options["push"]) == 'stable-security'
raise "Package built, but can't push to #{options["push"]}, use the 'release-security' subcommand"
end
repo.push pkg.name, version, options["push"]
end
end
end
desc "push PKG-NAME", "push a built package to a specified suite"
method_option :suite, :aliases => "-s", :type => :string,
:desc => "the target suite (defaults to testing)"
method_option :build, :aliases => "-b", :type => :string,
:desc => "which version to push (defaults to the highest one build)"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "force inclusion of the package to the suite"
def push(pkg_name)
repo = get_repo_handle
suite = nil
suite = options["suite"] if options.has_key? "suite"
if repo.codename_to_suite(options["suite"]) == 'stable-security'
raise "Can't push package to #{options["suite"]}, please use the 'release-security' subcommand"
end
version = nil
version = options["build"] if options.has_key? "build"
repo.push pkg_name, version, suite, options["force"] == true
end
desc "unpush PKG-NAME SUITE", "remove a built package from a suite"
def unpush(pkg_name, suite)
repo = get_repo_handle
repo.unpush pkg_name, suite
end
desc "list SUBCOMMAND [ARGS]", "show information about packages"
map "l" => :list, "ls" => :list
subcommand "list", List
desc "config SUBCOMMAND [ARGS]", "configure your repository"
map "c" => :config, "conf" => :config, "configure" => :config
subcommand "config", Conf
desc "rm [pkg-name]", "remove a package completely from the build system"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "force removal even if the package is still used"
def rm(pkg_name)
repo = get_repo_handle
repo.remove pkg_name, options["force"] == true
end
desc "rmbuild PKG-NAME VERSION", "remove a built version of a package"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "force removal even if the build is still used"
def rmbuild(pkg_name, version)
repo = get_repo_handle
repo.remove_build pkg_name, version, options["force"] == true
end
desc "update [SUITE]", "Update and rebuild (if necessary) all the packages in the suite"
method_option :branch, :aliases => "-b", :type => :string,
:desc => "Branch to use as the source for update"
def update(suite="testing")
log :info, "Updating all packages in the #{suite.fg "blue"} suite"
repo = get_repo_handle
branch = options["branch"] if options.has_key? "branch"
updated = 0
repo.list_packages(suite).each do |pkg|
log :info, "Updating #{pkg.name.style "pkg-name"}"
begin
version = pkg.build branch
rescue Dr::Package::UnableToBuild
log :info, ""
next
rescue Exception => e
# Handle all other exceptions and try to build next package
log :err, e.to_s
log :info, ""
next
end
if version && !repo.suite_has_higher_pkg_version?(suite, pkg, version)
repo.push pkg.name, version, suite
updated += 1
end
log :info, ""
end
log :info, "Updated #{updated.to_s.fg "blue"} packages in #{suite.fg "blue"}"
end
desc "git-tag-release TAG", "Mark relased packages' repositories"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "Force override existing tags"
method_option :package, :aliases => "-p", :type => :string,
:desc => "Only tag a single package"
method_option :summary, :aliases => "-s", :type => :string,
:desc => "A summary for the release (Github only)"
method_option :title, :aliases => "-t", :type => :string,
:desc => "A title for the release (Github only)"
def git_tag_release(tag)
repo = get_repo_handle
packages = if options["package"] == nil
repo.list_packages "stable"
else
if repo.get_subpackage_versions(options["package"])["stable"].empty?
log :warn, "This package isn't in the #{"stable".fg "green"} branch, skipping."
end
[repo.get_package(options["package"])]
end
packages.each do |pkg|
if pkg.is_a? Dr::GitPackage
version = repo.get_subpackage_versions(pkg.name)["stable"].values.max
bm = repo.get_build_metadata pkg.name, version
pkg.tag_release tag, bm["revision"], options
else
log :info, "#{pkg.name.style "pkg-name"} is not associated with a git repo, skipping"
end
end
end
desc "git-print-hashes", "Print the hashes for each repo in release"
method_option :package, :aliases => "-p", :type => :string,
:desc => "Only hash a single package"
def git_print_hashes()
repo = get_repo_handle
packages = if options["package"] == nil
repo.list_packages "stable"
else
if repo.get_subpackage_versions(options["package"])["stable"].empty?
log :warn, "This package isn't in the #{"stable".fg "green"} branch, skipping."
end
[repo.get_package(options["package"])]
end
packages.each do |pkg|
if pkg.is_a? Dr::GitPackage
version = repo.get_subpackage_versions(pkg.name)["stable"].values.max
if pkg.build_exists? version
bm = repo.get_build_metadata pkg.name, version
log :info, "#{pkg.get_repo_url} #{bm["revision"]}"
else
log :err, "#{pkg.get_repo_url} missing build version #{version}"
end
else
log :info, "#{pkg.name.style "pkg-name"} is not associated with a git repo, skipping"
end
end
end
desc "release RC-SUITE [DEST-SUITE]", "Push all the packages from RC-SUITE to DEST-SUITE (to release by default)"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "Force-push all released packages"
def release(rc_suite=nil, dest_suite="release")
if rc_suite.nil?
log :err, "#{"DEPRECATED".fg('red')}, suite required. Doing nothing."
log :err, "Use '#{"dr release RC-SUITE".fg("yellow")}' instead."
raise "Running '#{"dr release".fg("yellow")}' command without suite deprecated"
end
repo = get_repo_handle
suite = repo.codename_to_suite rc_suite
if suite.nil?
log :err, "Suite '#{rc_suite.fg("yellow")}' not found"
raise "Suite '#{rc_suite.fg("yellow")}' doesn't exist in the repo"
end
release_codename = dest_suite
release_suite = repo.codename_to_suite release_codename
log :info, "Pushing packages from #{rc_suite.fg("yellow")} to #{release_codename.fg("yellow")}"
suite_diff rc_suite, release_codename
prompt_to_confirm "Are you sure you want to continue?", "Aborting release"
log :info, "Releasing all packages from testing"
repo.list_packages(suite).each do |pkg|
v = repo.get_subpackage_versions(pkg.name)[suite].values
begin
repo.push pkg.name, v.max, release_suite, (options["force"] == true)
rescue Dr::AlreadyExists
;
end
end
log :info, "Removing packages that are not in #{rc_suite} any more"
repo.list_packages(release_codename).each do |pkg|
if ! repo.suite_has_package? suite, pkg.name
repo.unpush pkg.name, release_codename
end
end
end
desc "release-security PKG-NAME", "Push a built package from testing to 'stable-security' suite"
method_option :force, :aliases => "-f", :type => :boolean,
:desc => "Force-push the released package"
def release_security(pkg_name)
repo = get_repo_handle
suite_source = 'testing'
log :info, "Releasing pkg #{pkg_name.style "pkg-name"} package from '#{suite_source}' to 'stable-security'"
if !repo.suite_has_package? suite_source, pkg_name
log :err, "Package #{pkg_name.style "pkg-name"} not in '#{suite_source}'"
raise "Package #{pkg_name.style "pkg-name"} doesn't exist in the repo"
end
prompt_msg = "Are you absolutely sure you want to push package #{pkg_name.style "pkg-name"} to 'stable-security'?"
negative_msg = "Couldn't confirm for releasing to stable-security"
prompt_to_confirm prompt_msg, negative_msg
version = repo.get_subpackage_versions(pkg_name)[suite_source].values.max
log :info, "Package version #{version.style "version"} found in '#{suite_source}'"
begin
repo.push pkg_name, version, "stable-security", (options["force"] == true)
rescue Dr::AlreadyExists
;
end
end
desc "suite-diff SUITE OTHER-SUITE", "Show the differences between packages in two suites"
method_option :deb, :aliases => "-d", :type => :boolean,
:desc => "Check only deb packages"
def suite_diff(first, second)
repo = get_repo_handle
first = repo.codename_to_suite first
if !first
log :err, "Can't find the #{first.fg 'blue'} suite in this repo"
raise "Suite doesn't exist"
end
second = repo.codename_to_suite second
if !second
log :err, "Can't find the #{second.fg 'blue'} suite in this repo"
raise "Suite doesn't exist"
end
log :info, "Showing the differences between #{first.fg 'green'} and #{second.fg 'red'}"
if options["deb"]
log :info, "Only for Deb packages"
end
thread_pool repo.list_packages(first) do |pkg|
if options["deb"] and not pkg.is_a? Dr::DebPackage
next
end
subpackage_versions = repo.get_subpackage_versions pkg.name
first_v = subpackage_versions[first].values.max
if !repo.suite_has_package? second, pkg.name
log :info, "#{pkg.name.fg 'orange'} is in #{first.fg 'green'} but not in #{second.fg 'red'}"
next
end
second_v = subpackage_versions[second].values.max
if Dr::PkgVersion.new(first_v) != Dr::PkgVersion.new(second_v)
log :info, "#{pkg.name.fg 'orange'} #{first_v.fg 'green'} != #{second_v.fg 'red'}"
end
end
thread_pool repo.list_packages(second) do |pkg|
if !repo.suite_has_package? first, pkg.name
log :info, "#{pkg.name.fg 'orange'} is in #{second.fg 'red'} but not in #{first.fg 'green'}"
end
end
end
desc "force-sync PKG-NAME", "Force cloning the sources repository from scratch again"
method_option :url, :aliases => "-u", :type => :string,
:desc => "The URL to clone from"
method_option :branch, :aliases => "-b", :type => :string,
:desc => "The default branch to use for building"
def force_sync(pkg_name)
repo = get_repo_handle
pkg = repo.get_package pkg_name
if pkg.is_a? Dr::GitPackage
pkg.reinitialise_repo options["url"], options["branch"]
else
raise "The source of #{pkg_name.style "pkg-name"} is not managed by " +
"#{"dr".bright}"
end
end
#desc "snapshot", "save a snapshot of the archive"
#def snapshot(tag)
# repo = get_repo_handle
#end
desc "cleanup", "Remove builds beyond certain date or number"
method_option :package, :aliases => "-p", :type => :string,
:desc => "Cleanup this package only"
method_option :date, :aliases => "-d", :type => :string,
:desc => "Remove builds beyond this date (YYYYMMDD)"
method_option :number, :aliases => "-n", :type => :string,
:desc => "Keep only N newest builds"
def cleanup
repo = get_repo_handle
if options["date"] != nil && options["number"] != nil
log :err, "Can't use -n and -d at the same time"
raise "Bad arguments"
end
date = options["date"]
number = options["number"]
if options["date"] == nil && options["number"] == nil
number = 10
end
packages = unless options["package"] == nil
[repo.get_package(options["package"])]
else
repo.list_packages
end
packages.each do |pkg|
kept = 0
pkg.history.each do |version_string|
# Can't remove a used build
if repo.is_used? pkg.name, version_string
kept += 1
next
end
if date != nil
version = Dr::PkgVersion.new version_string
if version.date.to_i < date.to_i
rmbuild pkg.name, version_string
end
elsif number != nil && kept >= number.to_i
rmbuild pkg.name, version_string
else
kept += 1
end
end
end
end
desc "serve", "Start the archive server"
method_option :port, :aliases => "-p", :type => :numeric,
:desc => "The port to run the server on", :default => 80
method_option :bind, :aliases => "-b", :type => :string,
:desc => "Address to listen on", :default => "0.0.0.0"
method_option :route, :aliases => "-R", :type => :string,
:desc => "The route to serve the archive on", :default => "/"
def serve
repo = get_repo_handle
s = Dr::Server.new options["port"], options["route"],
options["bind"], repo.get_archive_path
s.start
end
private
def prompt_to_confirm(prompt_msg, negative_message)
response = 'x'
while ! ['y', 'n'].include? response
print prompt_msg
print "[y/n]: "
response = STDIN.gets.strip.downcase
if response == 'n'
log :err, "Replied negatively to prompt, aborting..."
raise negative_message
elsif response == 'y'
log :info, "Received confirmation, will carry on"
else
print "Not an acceptable answer, please answer y/n\n"
end
end
end
end
begin
Dr::check_dependencies [
"git", "reprepro", "gzip", "debuild", "debootstrap", "qemu-*-static",
"chroot", "curl", "gpg", "tar", "dpkg", "dpkg-deb", "dpkg-sig", "rm",
"sudo"
]
RepoCLI.start ARGV
rescue StandardError => e
Dr::Logger.log :err, e.to_s
e.backtrace.each do |line|
line = " #{line}" if line.length > 0 && line[0] == '/'
Dr::Logger.log :err, line.fg("grey")
end
end