make-cd

#!/usr/bin/perl
# ***** BEGIN LICENSE BLOCK *****
# Version: MPL 1.1/GPL 2.0/LGPL 2.1
#
# The contents of this file are subject to the Mozilla Public License Version
# 1.1 (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.mozilla.org/MPL/
#
# Software distributed under the License is distributed on an "AS IS" basis,
# WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
# for the specific language governing rights and limitations under the
# License.
#
# The Original Code is CD maker for Mozilla Store
#
# The Initial Developer of the Original Code is
# Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 2005
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#  Myk Melez <myk@mozilla.org>
#  J. Paul Reed <preed@sigkill.com>
#  Robert Helmer <robert@roberthelmer.com>
#
# Alternatively, the contents of this file may be used under the terms of
# either the GNU General Public License Version 2 or later (the "GPL"), or
# the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
# in which case the provisions of the GPL or the LGPL are applicable instead
# of those above. If you wish to allow use of your version of this file only
# under the terms of either the GPL or the LGPL, and not to allow others to
# use your version of this file under the terms of the MPL, indicate your
# decision by deleting the provisions above and replace them with the notice
# and other provisions required by the GPL or the LGPL. If you do not delete
# the provisions above, a recipient may use your version of this file under
# the terms of any one of the MPL, the GPL or the LGPL.
#
# ***** END LICENSE BLOCK *****

# A script for making a CD containing Mozilla product releases.
# Downsyncs builds from a build archive; copies builds and other files
# to a master directory, modifying paths and filenames along the way
# to conform to CD specifications; expands Linux installer tarballs
# as needed; and builds an ISO that can be burned to CD.

# In the future this script could also generate some of the other files
# from templates and burn the ISO to CD.

use strict;
use Cwd;
use File::Path;
use File::Copy;
use Data::Dumper;
use Getopt::Long;

################################################################################
# Script Configuration

# All variables declared with "our" so they can be overridden by the spec,
# which configures this script to build a particular CD.

# Whether or not to be chatty about what we're doing.
# Set this on the command line with --verbose or --noverbose.
our $VERBOSE = 1;

# The rsync host and module for the build archive.
our $RSYNC_HOST = "stage.mozilla.org";
our $RSYNC_MODULE = "mozilla-releases";

# Whether to use rsync -e ssh instead of the rsync protocol; uses $USER to
# construct the rsync line.
our $USE_SSH = 1;
our $SSH_RSYNC_PATH = '/home/ftp/pub/';

# The directory where we do our work; the current working directory by default.
# By default, we downsync builds into its archive/ subdirectory, look for other
# files in its other/ subdirectory, build the ISO master in its master/
# subdirectory, and generate the ISO file in it, but you can change all that
# by redefining the $ARCHIVE_DIR, $MASTER_DIR, $OTHER_DIR, and $ISO_FILE vars.
our $ROOT_DIR = cwd();

# The directory into which we put the builds we downsync from the build archive.
our $ARCHIVE_DIR = "$ROOT_DIR/archive";

# The directory containing other (non-build) files destined for the CD
# like the Windows autorun stuff and READMEs.
our $OTHER_DIR = "$ROOT_DIR/other";

# The directory where we put all the files we want on the CD in the hierarchy
# we want them in.  We build the ISO from this directory.
our $MASTER_DIR = "$ROOT_DIR/master";

# The name of the ISO file this script will generate.  Generally such names
# end in .iso.  This could include a path if you want to write the file to some
# place other than the root directory.
# Note: generally this should be redefined by the spec file.
our $ISO_FILE = "mozilla-release.iso";

# The application and volume IDs written to the CD.  According to the mkisofs
# docs, the application ID can be 128 characters and describes "the application
# that will be on the disc," while the volume ID can be 32 characters and is,
# among other things, "the name that is assigned to the disc on a Microsoft
# Win32 or Apple Mac platform."  According to an older version of this script,
# however, this value should be only sixteen characters or less, although
# the older version didn't say why.
# Note: generally this should be redefined by the spec file.
our $APP_AND_VOLUME_ID = "Mozilla Release";

# Which apps, versions, and locales we should put onto the CD.
# Note: for us to do anything, this must be defined by the spec file.
our $RELEASES = [];

# The file containing the spec, which configures this script to build
# a particular CD.  Set this on the command line with --spec [FILENAME].
my $SPEC_FILE;

# Get and parse the spec from the spec file.
GetOptions ( "spec=s"   => \$SPEC_FILE,
             "verbose!" => \$VERBOSE );
die "You must specify a spec file via --spec [FILENAME]\n" if !$SPEC_FILE;
my $return = do $SPEC_FILE;
die "couldn't parse $SPEC_FILE: $@" if $@;
die "couldn't do $SPEC_FILE: $!"    unless defined $return;
die "couldn't run $SPEC_FILE"       unless $return;


################################################################################
# Initialize

# Create the root, archive, and master directories if they don't already exist.
File::Path::mkpath($ROOT_DIR, $VERBOSE, 0755);
File::Path::mkpath($ARCHIVE_DIR, $VERBOSE, 0755);
File::Path::mkpath($MASTER_DIR, $VERBOSE, 0755);

# Change into the root directory.
chdir($ROOT_DIR);


################################################################################
# Downsync Builds

print "Downsyncing builds from archive.\n\n" if $VERBOSE;

foreach my $app (@$RELEASES) {
    my $dir = "$app->{archive_path}/$app->{version}";

    # Create the local destination directory if it doesn't already exist.
    # XXX Here and later we concatenate paths together with forward slashes,
    # but that's not portable to all platforms on which Perl runs.  We should
    # use File::Spec::catfile() instead, but I'm not sure it's compatible
    # with File::Path::mkpath(), which expects forward slashes.
    File::Path::mkpath("$ARCHIVE_DIR/$dir", $VERBOSE, 0755);

    # Download the release files into the local destination directory.
    # XXX This gets us way more than we actually need; can we instead
    # just download the files that matter (f.e. just those localizations
    # we're putting onto the CD, and only their installers)?

    my $sourceString;
    if ($USE_SSH) {
        $sourceString = $ENV{'USER'} . '@' . 
         "${RSYNC_HOST}:${SSH_RSYNC_PATH}/$dir/";
    } else {
        $sourceString = "${RSYNC_HOST}::${RSYNC_MODULE}/$dir/";
    }

    my @includedLocales;
    
    foreach my $locale (@{$app->{locales}}) {
        foreach my $platform ('/linux-i686/', '/mac/', '/unimac/', '/win32/') {
            push(@includedLocales, "--include");
            push(@includedLocales, join("", $platform, $locale, "/*"));
        }
    }
    
    my @args = (
        "-a", "--delete", "--delete-after",
        $USE_SSH ? ('-e', 'ssh') : (),
        $VERBOSE ? ("-v", "--progress") : "--quiet",
        "--include", '*/',
        @includedLocales,
        "--exclude", '*',
        $sourceString,                           # source
        "$ARCHIVE_DIR/$dir"                      # destination
    );

    system("rsync", @args) == 0 or die "rsync @args failed: $?";
}

print "Done downsyncing builds from archive.\n\n\n" if $VERBOSE;


################################################################################
# Sync Files to Master Directory

print "Syncing files to master directory.\n\n" if $VERBOSE;

foreach my $app (@$RELEASES) {
    my $from_root = "$ARCHIVE_DIR/$app->{archive_path}";
    foreach my $locale (@{$app->{locales}}) {
        foreach my $build (@{$app->{builds}}) {
            # Check if there are build-specific locales defined for this build;
            # if so, only include those locales for this build.  Note that all
            # build-specific locales must also be included in the application's
            # locale list.
            if (!$build->{locales} || grep($locale eq $_, @{$build->{locales}}))
            {
                sync_file($from_root, $app, $build->{from}, $build->{to},
                          $locale);
            }
        }
    }
    foreach my $other (@{$app->{others}}) {
        sync_file($OTHER_DIR, $app, $other->{from}, $other->{to});
    }
}

print "Done syncing files to master directory.\n\n\n" if $VERBOSE;

sub sync_file {
    my ($from_root, $app, $from, $to, $locale) = @_;

    # Generate source and destination file specs from templates.
    $from =~ s/%version%/$app->{version}/g;
    $from =~ s/%locale%/$locale/g if $locale;
    $to =~ s/%version%/$app->{version}/g;
    # Hackety hack hack. ja-JP-mac should go into the ja directory.
    $to =~ s/%locale%/$locale eq "ja-JP-mac" ? "ja" : $locale/ge if $locale;

    # Convert specs to absolute paths.
    $from = "$from_root/$from";
    $to = "$MASTER_DIR/$to";

    print "From: $from\n  To: $to\n\n" if $VERBOSE;

    # Extract parent directories from specs.
    # XXX Can we use the platform-independent File::Spec here?
    (my $from_dir = $from) =~ s/^(.*)(\/[^\/]*)$/$1/g;
    (my $to_dir = $to) =~ s/^(.*)(\/[^\/]*)$/$1/g;

    # Make sure the destination directory exists, and make its
    # modification time the same as the source directory's.
    system("mkdir", "-p", $to_dir) if !-e $to_dir;
    system("touch", "-r", $from_dir, $to_dir);

    # Sync the file from source to destination.
    # Note: some locales don't exist for all files (f.e. ja-JP-mac
    # only exists for the Mac build), so we ignore rsync errors.
    system("rsync", "-a", ($VERBOSE ? "-v" : ()), $from, $to);

    print "\n" if $VERBOSE;
}


################################################################################
# Expand Linux Installer Tarball

# The Linux installer is a tarball, which would be cumbersome to install,
# since users can't just expand it from the CD with a simple "tar -xvzf",
# because the CD is write-only.  And even if they could, it means an extra
# step, and possibly a complicated one if they don't understand command-line
# tools and don't have a GUI tarball expander.  Thus, to make things easier,
# we expand the tarball before writing the installer to CD.  Then Linux users
# can just run the installer from the CD just like Mac and Windows users.

print "Expanding Linux installer tarballs...\n\n" if $VERBOSE;

foreach my $app (@$RELEASES) {
    if ($app->{linux_dest} && $app->{linux_name}) {
        my $version = $app->{version};
        (my $dest = $app->{linux_dest}) =~ s/%version%/$version/g;
        my $name = $app->{linux_name};
        foreach my $locale (@{$app->{locales}}) {
            my $dir = "$MASTER_DIR/$dest/$locale";
            my $file = "$name-$version.installer.tar.gz";
            if (-e "$dir/$file") {
                print "Expanding $dir/$file.\n" if $VERBOSE;

                chdir($dir);
                system("tar", "-x", "-z", ($VERBOSE ? "-v" : ()), "-f", $file);
                # We've expanded the tarball into appname-installer, but we
                # really want it to be appname-version-installer, so we move
                # it there, but first we have to remove an existing
                # appname-version-installer directory, if any, so that moving
                # appname-installer to it doesn't fail.
                rmtree("$name-$version-installer", $VERBOSE, 1);
                move("$name-installer", "$name-$version-installer")
                  or die("couldn't rename $name-installer " .
                         "to $name-$version-installer: $!");
                # Don't want the tarball anymore.
                # XXX Can tar do this automagically?
                unlink($file);
                chdir($ROOT_DIR);

                print "Done expanding $dir/$file.\n" if $VERBOSE;
            }
            else {
                print "Not expanding $dir/$file: doesn't exist.\n" if $VERBOSE;
            }
        }
    }
}

print "Done expanding Linux installer tarballs.\n\n\n" if $VERBOSE;


################################################################################
# Build ISO

print "Building ISO.\n\n" if $VERBOSE;

system("mkisofs", "-o", $ISO_FILE, "-J", "-R", "-A", $APP_AND_VOLUME_ID,
       "-V", $APP_AND_VOLUME_ID, $VERBOSE ? "-v" : "--quiet", $MASTER_DIR);

print "Done building ISO.\n\n\n" if $VERBOSE;


################################################################################
# Burn ISO to CD

# XXX Not done, but maybe should be preffed off by default until this script
# is sufficiently mature that it can be relied upon to work right the first time
# most of the time.