Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Fetching contributors…

Cannot retrieve contributors at this time

executable file 572 lines (506 sloc) 16.133 kb
#!/usr/bin/perl -w
use strict;
use Cwd;
# Usage:
#
# ./sync-all [-q] [-s] [--ignore-failure] [-r repo]
# [--nofib] [--testsuite] [--checked-out] cmd [git flags]
#
# Applies the command "cmd" to each repository in the tree.
# sync-all will try to do the right thing for both git and darcs repositories.
#
# e.g.
# ./sync-all -r http://darcs.haskell.org/ghc get
# To get any repos which do not exist in the local tree
#
# ./sync-all pull
# To pull everything from the default repos
#
# -------------- Flags -------------------
# -q says to be quite, and -s to be silent.
#
# --ignore-failure says to ignore errors and move on to the next repository
#
# -r repo says to use repo as the location of package repositories
#
# --checked-out says that the remote repo is in checked-out layout, as
# opposed to the layout used for the main repo. By default a repo on
# the local filesystem is assumed to be checked-out, and repos accessed
# via HTTP or SSH are assumed to be in the main repo layout; use
# --checked-out to override the latter.
#
# --nofib, --testsuite also get the nofib and testsuite repos respectively
#
# ------------ Which repos to use -------------
# sync-all uses the following algorithm to decide which remote repos to use
#
# It always computes the remote repos from a single base, $repo_base
# How is $repo_base set?
# If you say "-r repo", then that's $repo_base
# otherwise $repo_base is set by asking git where the ghc repo came
# from, and removing the last component (e.g. /ghc.git/ of /ghc/).
#
# Then sync-all iterates over the package found in the file
# ./packages; see that file for a description of the contents.
#
# If $repo_base looks like a local filesystem path, or if you give
# the --checked-out flag, sync-all works on repos of form
# $repo_base/<local-path>
# otherwise sync-all works on repos of form
# $repo_base/<remote-path>
# This logic lets you say
# both sync-all -r http://darcs.haskell.org/ghc-6.12 pull
# and sync-all -r ../HEAD pull
# The latter is called a "checked-out tree".
# NB: sync-all *ignores* the defaultrepo of all repos other than the
# root one. So the remote repos must be laid out in one of the two
# formats given by <local-path> and <remote-path> in the file 'packages'.
$| = 1; # autoflush stdout after each print, to avoid output after die
my $defaultrepo;
my @packages;
my $verbose = 2;
my $ignore_failure = 0;
my $checked_out_flag = 0;
my $get_mode;
my %tags;
# Figure out where to get the other repositories from.
sub getrepo {
my $basedir = ".";
my $repo;
if (defined($defaultrepo)) {
$repo = $defaultrepo;
chomp $repo;
} else {
# Figure out where to get the other repositories from,
# based on where this GHC repo came from.
my $branch = `git branch | grep "\* " | sed "s/^\* //"`; chomp $branch;
my $remote = `git config branch.$branch.remote`; chomp $remote;
$repo = `git config remote.$remote.url`; chomp $repo;
}
my $repo_base;
my $checked_out_tree;
if ($repo =~ /^...*:/) {
# HTTP or SSH
# Above regex says "at least two chars before the :", to avoid
# catching Win32 drives ("C:\").
$repo_base = $repo;
# --checked-out is needed if you want to use a checked-out repo
# over SSH or HTTP
if ($checked_out_flag) {
$checked_out_tree = 1;
} else {
$checked_out_tree = 0;
}
# Don't drop the last part of the path if specified with -r, as
# it expects repos of the form:
#
# http://darcs.haskell.org
#
# rather than
#
# http://darcs.haskell.org/ghc
#
if (!$defaultrepo) {
$repo_base =~ s#/[^/]+/?$##;
}
}
elsif ($repo =~ /^\/|\.\.\/|.:(\/|\\)/) {
# Local filesystem, either absolute or relative path
# (assumes a checked-out tree):
$repo_base = $repo;
$checked_out_tree = 1;
}
else {
die "Couldn't work out repo";
}
return $repo_base, $checked_out_tree;
}
sub parsePackages {
my @repos;
my $lineNum;
open IN, "< packages" or die "Can't open packages file";
@repos = <IN>;
close IN;
@packages = ();
$lineNum = 0;
foreach (@repos) {
chomp;
$lineNum++;
if (/^([^# ]+) +([^ ]+) +([^ ]+) +([^ ]+)$/) {
my %line;
$line{"localpath"} = $1;
$line{"tag"} = $2;
$line{"remotepath"} = $3;
$line{"vcs"} = $4;
push @packages, \%line;
}
elsif (! /^(#.*)?$/) {
die "Bad content on line $lineNum of packages file: $_";
}
}
}
sub message {
if ($verbose >= 2) {
print "@_\n";
}
}
sub warning {
if ($verbose >= 1) {
print "warning: @_\n";
}
}
sub scm {
my $dir = shift;
my $scm = shift;
my $pwd;
if ($dir eq '.') {
message "== running $scm @_";
} else {
message "== $dir: running $scm @_";
$pwd = getcwd();
chdir($dir);
}
system ($scm, @_) == 0
or $ignore_failure
or die "$scm failed: $?";
if ($dir ne '.') {
chdir($pwd);
}
}
sub scmall {
my $command = shift;
my $localpath;
my $tag;
my $remotepath;
my $scm;
my $line;
my $branch_name;
my $subcommand;
my $path;
my $wd_before = getcwd;
my $pwd;
my @args;
my ($repo_base, $checked_out_tree) = getrepo();
my $is_github_repo = $repo_base =~ m/(git@|git:\/\/|https:\/\/)github.com/;
parsePackages;
@args = ();
if ($command =~ /^remote$/) {
while (@_ > 0 && $_[0] =~ /^-/) {
push(@args,shift);
}
if (@_ < 1) { help(); }
$subcommand = shift;
if ($subcommand ne 'add' && $subcommand ne 'rm' && $subcommand ne 'set-url') {
help();
}
while (@_ > 0 && $_[0] =~ /^-/) {
push(@args,shift);
}
if (($subcommand eq 'add' || $subcommand eq 'rm') && @_ < 1) {
help();
} elsif (@_ < 1) { # set-url
$branch_name = 'origin';
} else {
$branch_name = shift;
}
} elsif ($command eq 'new') {
if (@_ < 1) {
$branch_name = 'origin';
} else {
$branch_name = shift;
}
}
push(@args, @_);
for $line (@packages) {
$localpath = $$line{"localpath"};
$tag = $$line{"tag"};
$remotepath = $$line{"remotepath"};
$scm = $$line{"vcs"};
# Check the SCM is OK as early as possible
die "Unknown SCM: $scm" if (($scm ne "darcs") and ($scm ne "git"));
# We can't create directories on GitHub, so we translate
# "package/foo" into "package-foo".
if ($is_github_repo) {
$remotepath =~ s/\//-/;
}
# Work out the path for this package in the repo we pulled from
if ($checked_out_tree) {
$path = "$repo_base/$localpath";
}
else {
$path = "$repo_base/$remotepath";
}
if ($command =~ /^(?:g|ge|get)$/) {
# Skip any repositories we have not included the tag for
if (not defined($tags{$tag})) {
$tags{$tag} = 0;
}
if ($tags{$tag} == 0) {
next;
}
if (-d $localpath) {
warning("$localpath already present; omitting")
if $localpath ne ".";
if ($scm eq "git") {
scm ($localpath, $scm, "config", "core.ignorecase", "true");
}
next;
}
# Note that we use "." as the path, as $localpath
# doesn't exist yet.
if ($scm eq "darcs") {
# The first time round the loop, default the get-mode
if (not defined($get_mode)) {
warning("adding --partial, to override use --complete");
$get_mode = "--partial";
}
scm (".", $scm, "get", $get_mode, $path, $localpath, @args);
}
else {
scm (".", $scm, "clone", $path, $localpath, @args);
scm ($localpath, $scm, "config", "core.ignorecase", "true");
}
next;
}
if (-d "$localpath/_darcs") {
if (-d "$localpath/.git") {
die "Found both _darcs and .git in $localpath";
}
$scm = "darcs";
} elsif (-d "$localpath/.git") {
$scm = "git";
} elsif ($tag eq "") {
die "Required repo $localpath is missing";
} else {
message "== $localpath repo not present; skipping";
next;
}
# Work out the arguments we should give to the SCM
if ($command =~ /^(?:w|wh|wha|what|whats|whatsn|whatsne|whatsnew|status)$/) {
if ($scm eq "darcs") {
$command = "whatsnew";
}
elsif ($scm eq "git") {
$command = "status";
}
else {
die "Unknown scm";
}
# Hack around 'darcs whatsnew' failing if there are no changes
$ignore_failure = 1;
scm ($localpath, $scm, $command, @args);
}
elsif ($command =~ /^commit$/) {
# git fails if there is nothing to commit, so ignore failures
$ignore_failure = 1;
scm ($localpath, $scm, "commit", @args);
}
elsif ($command =~ /^(?:pus|push)$/) {
scm ($localpath, $scm, "push", @args);
}
elsif ($command =~ /^(?:pul|pull)$/) {
scm ($localpath, $scm, "pull", @args);
}
elsif ($command =~ /^(?:s|se|sen|send)$/) {
if ($scm eq "darcs") {
$command = "send";
}
elsif ($scm eq "git") {
$command = "send-email";
}
else {
die "Unknown scm";
}
scm ($localpath, $scm, $command, @args);
}
elsif ($command =~ /^fetch$/) {
scm ($localpath, $scm, "fetch", @args);
}
elsif ($command =~ /^new$/) {
my @scm_args = ("log", "$branch_name..");
scm ($localpath, $scm, @scm_args, @args);
}
elsif ($command =~ /^log$/) {
scm ($localpath, $scm, "log", @args);
}
elsif ($command =~ /^remote$/) {
my @scm_args;
$ignore_failure = 1;
if ($subcommand eq 'add') {
@scm_args = ("remote", "add", $branch_name, $path);
} elsif ($subcommand eq 'rm') {
@scm_args = ("remote", "rm", $branch_name);
} elsif ($subcommand eq 'set-url') {
@scm_args = ("remote", "set-url", $branch_name, $path);
}
scm ($localpath, $scm, @scm_args, @args);
}
elsif ($command =~ /^checkout$/) {
# Not all repos are necessarily branched, so ignore failure
$ignore_failure = 1;
scm ($localpath, $scm, "checkout", @args)
unless $scm eq "darcs";
}
elsif ($command =~ /^grep$/) {
# Hack around 'git grep' failing if there are no matches
$ignore_failure = 1;
scm ($localpath, $scm, "grep", @args)
unless $scm eq "darcs";
}
elsif ($command =~ /^clean$/) {
scm ($localpath, $scm, "clean", @args)
unless $scm eq "darcs";
}
elsif ($command =~ /^reset$/) {
scm ($localpath, $scm, "reset", @args)
unless $scm eq "darcs";
}
elsif ($command =~ /^config$/) {
scm ($localpath, $scm, "config", @args)
unless $scm eq "darcs";
}
else {
die "Unknown command: $command";
}
}
}
sub help()
{
# Get the built in help
my $help = <<END;
Usage:
./sync-all [-q] [-s] [--ignore-failure] [-r repo] [--checked-out]
[--package-tag1] ... [--package-tagN] command
Supported commands:
* whatsnew
* commit
* push
* pull
* get, with options:
* --<package-tag>
* --complete
* --partial
* fetch
* send
* new
* remote add <remote-name>
* remote rm <remote-name>
* remote set-url [--push] <remote-name>
* checkout
* grep
* clean
* reset
* config
* log
Available package-tags are:
END
# Collect all the tags in the packages file
my %available_tags;
open IN, "< packages" or die "Can't open packages file";
while (<IN>) {
chomp;
if (/^([^# ]+) +(?:([^ ]+) +)?([^ ]+) +([^ ]+)/) {
if (defined($2) && $2 ne "-") {
$available_tags{$2} = 1;
}
}
elsif (! /^(#.*)?$/) {
die "Bad line: $_";
}
}
close IN;
# Show those tags and the help text
my @available_tags = keys %available_tags;
print "$help@available_tags\n";
exit 1;
}
sub main {
if (! -d ".git" || ! -d "compiler") {
die "error: sync-all must be run from the top level of the ghc tree."
}
$tags{"-"} = 1;
$tags{"dph"} = 1;
while ($#_ ne -1) {
my $arg = shift;
# We handle -q here as well as lower down as we need to skip over it
# if it comes before the source-control command
if ($arg eq "-q") {
$verbose = 1;
}
elsif ($arg eq "-s") {
$verbose = 0;
}
elsif ($arg eq "-r") {
$defaultrepo = shift;
}
elsif ($arg eq "--ignore-failure") {
$ignore_failure = 1;
}
elsif ($arg eq "--complete" || $arg eq "--partial") {
$get_mode = $arg;
}
# Use --checked-out if the remote repos are a checked-out tree,
# rather than the master trees.
elsif ($arg eq "--checked-out") {
$checked_out_flag = 1;
}
# --<tag> says we grab the libs tagged 'tag' with
# 'get'. It has no effect on the other commands.
elsif ($arg =~ m/^--no-(.*)$/) {
$tags{$1} = 0;
}
elsif ($arg =~ m/^--(.*)$/) {
$tags{$1} = 1;
}
else {
unshift @_, $arg;
if (grep /^-q$/, @_) {
$verbose = 1;
}
last;
}
}
if ($#_ eq -1) {
help();
}
else {
# Give the command and rest of the arguments to the main loop
scmall @_;
}
}
END {
my $ec = $?;
my $pwd = getcwd();
message "== Checking for old haddock repo";
if (-d "utils/haddock/.git") {
chdir("utils/haddock");
if ((system "git log -1 87e2ca11c3d1b1bc49900fba0b5c5c6f85650718 > /dev/null 2> /dev/null") == 0) {
print <<EOF;
============================
ATTENTION!
You have an old haddock repository in your GHC tree!
Please remove it (e.g. "rm -r utils/haddock"), and then run
"./syncs-all get" to get the new repository.
============================
EOF
}
chdir($pwd);
}
message "== Checking for old binary repo";
if (-d "libraries/binary/.git") {
chdir("libraries/binary");
if ((system "git log -1 749ac0efbde3b14901417364a872796598747aaf > /dev/null 2> /dev/null") == 0) {
print <<EOF;
============================
ATTENTION!
You have an old binary repository in your GHC tree!
Please remove it (e.g. "rm -r libraries/binary"), and then run
"./syncs-all get" to get the new repository.
============================
EOF
}
chdir($pwd);
}
$? = $ec;
}
main(@ARGV);
Jump to Line
Something went wrong with that request. Please try again.