install_packages

#!/usr/bin/perl

#*********************************************************************
#
# install_packages -- read package config and install packages via apt-get
#
# This script is part of FAI (Fully Automatic Installation)
# (c) 2000-2016, Thomas Lange, lange@informatik.uni-koeln.de
# (c) 2003-2004, Henning Glawe, glaweh@physik.fu-berlin.de
# (c) 2004     , Jonas Hoffmann, jhoffman@physik.fu-berlin.de
# PRELOAD feature from Thomas Gebhardt  <gebhardt@hrz.uni-marburg.de>
#
#*********************************************************************
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; see the file COPYING. If not, write to the
# Free Software Foundation, Inc., 59 Temple Place - Suite 330, Boston,
# MA 02111-1307, USA.
#*********************************************************************

$0=~ s#.+/##; # remove path from program name

# import variables: $verbose, $MAXPACKAGES, $classes, $FAI, $FAI_ROOT, $aptoptions, $FAI_DEBSOURCESDIR

use strict;
use Getopt::Std;

# global variables
our ($opt_d,$opt_l,$opt_L,$opt_v,$opt_h,$opt_H,$opt_m,$opt_n,$opt_N,$opt_p,$opt_s);
my $listonly; # flag, that indicates that only a list of packages will be printed
our %command;
our $atype;    # type of action (for apt-get, aptitude,..) that will be performed
my $dryrun=0;
my $verbose;
my $FAI_ROOT;
my @classes;
my $classpath;
my $rootcmd;
my @preloadlist;
my @preloadrmlist;
my $_config;
my $_system;
my $hasdebian=0;  # some Debian related commands/types are used in package_config
my %list;   # hash of arrays, key=type (yumi,aptitude,..), list of packages
my %types;  # hash containing the types found in all loaded package_config files
my %classisdef;
my $maxpl;  # maximum length of package list
my $cache;  # instance of AptPkg::Cache
my @unknown; # unknown packages
my @known;   # list of all known packages
my $execerrors; # counts execution errors
my $use_aptpkg=0;
my $aptopt='-y -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confnew';
my $downloaddir="/var/cache/apt/archives/partial/"; # where to download packages that gets only unpacked
my $debsourcesdir='/var/lib/fai/packages';

$| = 1;


# @commands is the order of the commands that are executed
our @commands = qw/y2i y2r zypper zypper-rm yast rpmr urpmi urpme yumgroup yumi yumr dnfgroup dnfi dnfr smarti smartr hold taskrm taskinst clean-internal cupt cupt-r aptitude aptitude-r install unpack remove dselect-upgrade/;
%command = (
          "install" => "apt-get $aptopt --fix-missing install",
    "inst-internal" => "apt-get $aptopt --fix-missing -s install",
           "remove" => "apt-get $aptopt --purge remove",
  "dselect-upgrade" => "apt-get $aptopt dselect-upgrade",
         "taskinst" => "tasksel install",
           "taskrm" => "tasksel remove",
             "hold" => "dpkg --set-selections",
   "clean-internal" => "apt-get clean",
         "aptitude" => "aptitude -R $aptopt install",
       "aptitude-r" => "aptitude -r $aptopt install",
              "apt" => "apt $aptopt install",
             "cupt" => "cupt -R $aptopt install",
           "cupt-r" => "cupt $aptopt install",
           "unpack" => "cd $downloaddir; aptitude download",
  "unpack-internal" => "dpkg --unpack --recursive $downloaddir; rm $downloaddir/*.deb",
 "pending-internal" => "dpkg --configure --pending",
   "dpkgc-internal" => "dpkg -C",
            "urpmi" => "urpmi --auto --foce --allow-force --keep",
            "urpme" => "urpme --auto --foce --allow-force --keep",
             "yumi" => "yum -y install",
             "yumr" => "yum -y remove",
        "yumgroup"  => "yum -y groupinstall",
             "dnfi" => "dnf -y install",
             "dnfr" => "dnf -y remove",
        "dnfgroup"  => "dnf -y group install",
             "y2i"  => "y2pmsh isc",
             "y2r"  => "y2pmsh remove",
             "yast" => "yast -i",
           "zypper" => "zypper -n install",
        "zypper-rm" => "zypper -n remove",
             "rpmr" => "rpm -e",
           "smarti" => "smart install -y",
           "smartr" => "smart remove -y",
  "smartc-internal" => "smart clean",
);

getopts('dhHvlLm:p:nNs');

$opt_N || check_aptpkg(); # do not use AptPkg when -N is given

$listonly = $opt_l || $opt_L;
$opt_h && usage();
$opt_H && showcommands();
$dryrun=1 if $opt_n;
$verbose=$ENV{verbose} || $opt_v;
$opt_d && setdownloadonly();
$maxpl=$ENV{MAXPACKAGES} || $opt_m ;
$maxpl && $verbose && warn "Maximum number of packages installed at a time set to $maxpl\n";
$maxpl or $maxpl = 99 ; # set default value

my $qopt="-qq" unless $verbose;
my $devnull=">/dev/null" unless $verbose;

$FAI_ROOT = $ENV{FAI_ROOT};
$classpath = $opt_p || "$ENV{FAI}/package_config";
$rootcmd = ($FAI_ROOT eq "/" ) ? '' : "chroot $FAI_ROOT";
@classes = grep { !/^#|^\s*$/ } split(/[\s\n]+/,$ENV{classes});
foreach (@classes) { $classisdef{$_}=1;}

warn "$0: reading config files from directory $classpath\n" if $verbose;
foreach (@classes) {
  &readconfig($classpath,$_) if -f "$classpath/$_"; # read all package config files
}

# check if any Debian related commands/types are used in package_config
my @debiantypes= qw/taskinst cupt cupt-r apt aptitude aptitude-r install remove dselect-upgrade smarti/;
foreach my $dt (@debiantypes) {
  $types{$dt} and $hasdebian=1;
}

# use AptPkg only if target uses apt and Apt::Pkg is available
if ($use_aptpkg && $hasdebian) {
  $_config->init;                 # initialize AptPkg
  $_config->set("Dir",$FAI_ROOT); # simulate "chroot"
  $_config->{quiet}=2;            # don't show cache initialization messages
  $_system = $_config->system;
  $cache = new AptPkg::Cache;
}

if ($types{'smarti'}) {  # smarti is used in a packages_config file
  $command{'clean-internal'} = $command{'smartc-internal'};
  $command{'pending-internal'} = "true";
  $command{'dpkgc-internal'} = "true";
}

# get files which must exist before installing packages
foreach my $entry (@preloadlist,@preloadrmlist) {
  my ($url, $directory) = @$entry;
  if ($url =~ m!^file:/(.+)!) {
    my $file = $1;
    execute("cp $FAI_ROOT/$file $FAI_ROOT/$directory") unless $listonly;
  } else {
    execute("cd $FAI_ROOT/$directory; curl -# -O $url") unless $listonly;
  }
}

# - - - - - - - - - - - - - - - - - - - - - - - - - - -
# begin of the big foreach loop
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
# call apt-get or tasksel for each type of command whith the list of packages
foreach $atype (@commands) {

  if ($atype eq "clean-internal" && $hasdebian) {
    execute("$rootcmd $command{'clean-internal'}") unless $listonly;
    next;
  }

  # skip if empty list
  next unless defined $list{$atype};

  if ($atype eq "dselect-upgrade") {
    dselectupgrade($atype);
    next;
  }

  my $packlist = join(' ',@{$list{$atype}});

  if ($atype eq "hold") {
    my $hold = join " hold\n", @{$list{hold}}, "";
    execute("echo \"$hold\" | $rootcmd $command{hold}");
    next;
  }

  if ($atype eq "install" || $atype eq "smarti" || $atype eq "cupt"|| $atype eq "cupt-r"|| $atype eq "apt" || $atype eq "aptitude" || $atype eq "aptitude-r" || $atype eq "unpack" || $opt_l || $opt_L) {

    mkpackagelist(@{$list{$atype}}); # create lists of known and unknown packages
    getsources(); # retrieve sources
    if ($opt_l) {
      # only print the package list
      print join ' ',@known,"\n";
      exit 0;
    }
    if ($opt_L) {
      # only print the package list
      execute("$rootcmd $command{'inst-internal'} @known | egrep ^Inst");
      exit 0;
    }
    # pass only maxpl packages to apt-get
    while (@known) {
      my $shortlist = join(' ', splice @known,0,$maxpl);
      # print "SL $shortlist\n";
      execute("$rootcmd $command{$atype} $shortlist") if $shortlist;
      execute("$rootcmd $command{'clean-internal'}") if $hasdebian;
      execute("$rootcmd $command{'unpack-internal'}") if ($atype eq "unpack"); # unpack and rm deb files
    }
    next;
  }

  if ($atype eq "taskinst" || $atype eq "taskrm") {
    foreach my $tsk (@{$list{$atype}}) {
      execute("$rootcmd $command{$atype} $tsk");
    }
    next;
  }

  # other types
  execute("$rootcmd $command{$atype} $packlist") if $packlist;
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
# end of the big foreach loop
# - - - - - - - - - - - - - - - - - - - - - - - - - - -

# remove preloaded files
foreach my $entry (@preloadrmlist) {
  my ($url, $directory) = @$entry;
  $url =~ m#/([^/]+$)#;
  my $file =  "$directory/$1";
  print "rm $file\n" if $verbose;
  unlink $file || warn "Can't remove $file\n";
}

if ($hasdebian) {
  # in case of unconfigured packages because of apt errors
  # retry configuration
  execute("$rootcmd $command{'pending-internal'}");
  # check if all went right
  execute("$rootcmd $command{'dpkgc-internal'}");
  # clean apt cache
  execute("$rootcmd $command{'clean-internal'}");
}

if ($execerrors) {
  warn "$execerrors errors during executing of $0\n";
  exit 3;
}

exit 0; # end of program
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub readconfig {

  my ($path,$file) = @_;
  my ($package,$type,$cllist,@oclasses,$doit);

  open (FILE,"$path/$file") || warn "ERROR $0: Can't read config file: $path/$file\n";
  warn "$0: read config file $file\n" if $verbose;

  while (<FILE>) {
    next if /^#/;    # skip comments
    s/#.*$//;        # delete comments
    next if /^\s*$/; # skip empty lines
    chomp;
    /^PRELOAD\s+(\S+)\s+(\S+)/   and push(@preloadlist,   [$1,$2]),next;
    /^PRELOADRM\s+(\S+)\s+(\S+)/ and push(@preloadrmlist, [$1,$2]),next;

    if (/^PACKAGES\s+(\S+)\s*/) {
      ($type,$cllist) = ($1,$');
      warn "WARNING: Unknow action $type after PACKAGES\n" unless defined $command{$type};
      # by default no classes are listed after this command so doit
      $doit = 1;
      if ($cllist) {
        # no classes specified after PACKAGES command
        # so add all packages listed
        # use packages on for a list of classes
        $doit = 0; # assume no class is defined
        @oclasses = split(/\s+/,$cllist);
        # if a listed class is defined, add the packaes, otherwise skip these packages
        foreach (@oclasses) { exists $classisdef{$_} and $doit = 1;}
      }
      next;
    }

    # warning if uppercase letters are found (package are always lowercase)
    warn "WARNING: Uppercase character found in package name in line $_\n" if $_ =~ /[A-Z]/;
    unless ($type) {
        warn "ERROR: PACKAGES .. line missing in $file\n";
        next;
    }
    push @{$list{$type}}, split if $doit;
    $types{$type}=1 if $doit;   # remember which types are used in package_config
  }
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub execute {

  # execute a command or only print it
  my @cmds = @_;

  # TODO: split cmds into array except when a pipe is found

  my $command = join (' ',@cmds);
  my $error;

  $dryrun and $verbose = 1;
  $verbose and warn "$0: executing $command\n";
  $dryrun and return;

  # @cmds should me more efficient
  $error = system @cmds;
  warn "ERROR: $error $?\n" if $error;
  my $rc = $?>>8;
  warn "ERROR: $cmds[0] return code $rc\n" if $rc;
  $execerrors++ if $rc;
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub dselectupgrade {

  my $type = shift;
  my ($package,$action,$list);
  my $tempfile = "$FAI_ROOT/var/lib/fai/dpkg-selections.tmp"; # TODO: use better uniq filename
  while (@{$list{$type}}) {
    $package = shift @{$list{$type}};
    $action  = shift @{$list{$type}};
    $list .= "$package $action\n";
  }

  open TMP,"> $tempfile" || die " Can't write to $tempfile";
  print TMP $list;
  close TMP;

  execute("$rootcmd dpkg --set-selections < $tempfile");
  execute("$rootcmd $command{$type}");
  unlink $tempfile;
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub installable {

  # ckeck, wether package can be installed.
  # return false for packages which only exist in
  # dpkg status files

  my $package  = shift;
  # print "Checking wether $package is installable...\n";

  # version list of package
  my $vlist = $cache->{$package}{VersionList};

  # for each version
  foreach my $version (@$vlist) {
    # list of package files
    my $flist = $version->{FileList};
    foreach my $fitem (@$flist) {
      # if the file is a package index (not a dpkg status file)
      # the package is installable
      return 1 if ($fitem->{File}{IndexType} eq "Debian Package Index");
    }
  }

  # no package index file found for the current package
  return 0;
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub mkpackagelist {

  my @complete = @_; # copy original list
  my @orig = ();
  @unknown = @known = ();
  my %complete = ();

  # create two list of packages
  # @unknown contains the unknown packages
  # @known contains the known packages

  # nothing to do, when no Debian types/commands are used at all
  unless ($hasdebian) {
    @known = @complete;
    writepackages();
    return;
  }

  # no package name checking when using smart
  if ($atype eq "smarti" || $atype eq "smartr") {
    @known=@complete;
    writepackages();
    return;
  }

  # no package name checking when -N was given
  if ($opt_N) {
    @known=@complete;
    writepackages();
    return;
  }

  # ignore packages ending with - when using -d
  @complete = grep {!/-$/} @complete if $opt_d;

  foreach (reverse @complete) {
    my $pack = $_;
    if ( /^(.+)[_=+-]$/ and $1 and $cache->exists($1)) {
      $pack = $1;
    }
    if (! defined ($complete{$pack})) {
      $complete{$pack} = 1;
      unshift @orig, $_;
    }
  }

  # currently we disable the complete checking of package names if aptitude
  # is used. This is because we can't check task names, which are supported by aptitude
  if ($atype eq "aptitude" || $atype eq "aptitude-r") {
    @known=@orig;
    writepackages();
    return;
  }

  foreach my $pack (@orig) {

#     if ($atype eq "aptitude" && $pack =~ /^~/) {
#       # add to known if is it an aptitude search pattern
#       push @known, $pack;
#       next;
#     }
# TODO: aptitude also can install tasks. How do we check them? Currently they will push to @unknown

    if ($cache->exists($pack)) { # simple package name
      # check for packages with no installation candidates
      # explicitely don't check if the name is provided by some other package:
      # apt-get install <virtual> fails with newer apt

      if (installable($pack)) {
        push @known, $pack;
      } else {
        push @unknown, $pack;
      }
    }

    # simulate APTs and aptitudes understanding of "special" package names
    # suffixed by -, +
    elsif ( $pack =~ /^(.+?)(?:\/.+|[_=+-])$/ and $cache->exists($1)) {

      if (defined ($cache->{$1}{VersionList})) {
        push @known, $pack;
      } else {
        push @unknown, $pack;
      }

    } else {
      push @unknown, $pack;
    }
  }

  writepackages();
  warn "WARNING: These unknown packages are removed from the installation list: " . join(' ',sort @unknown) . "\n" if @unknown;
}

# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub writepackages {

  # write package list to log file

  return if $opt_d; # do not write the list if we only download packages
  return if $opt_l; # do not write the list to a file, just print it

  open(LIST,"> $FAI_ROOT/var/log/install_packages.list") || warn "ERROR $0: Can't write package list file: $!\n";
  print LIST "# List of all packages that will be installed via install_packages\n";
  for (@known) { print LIST "$_\n"; }
  close(LIST);
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
# download debian source packages
sub getsources {

  return unless $opt_s;

  print "Trying to retrieve sources as specified via -s option for install_packages\n" if $verbose;

  if (!$ENV{'FAI_DEBSOURCESDIR'}) {
    die "Error: FAI_DEBSOURCESDIR is not set, can not retrieve sources.";
  }

  execute("$rootcmd sh -c \"mkdir -p $debsourcesdir;\"");
  execute("mkdir -p $ENV{'FAI_DEBSOURCESDIR'};");
  for (@known) {
    execute("$rootcmd sh -c \"cd $debsourcesdir ; apt-get --download-only source $_\"");
    execute("mv $FAI_ROOT/$debsourcesdir/* $ENV{'FAI_DEBSOURCESDIR'}");
  }
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub usage {

  print << "EOF";
install_packages

 Please read the manual pages install_packages(8).
EOF
  exit 0;
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub setdownloadonly {

# Definitions for install_packages(8) for download only mode
# Used by fai-mirror(1)

# we are using a variable from %ENV
# for debugging remove >/dev/null and -q

undef @commands;
undef %command;

$maxpl=9999;
@commands = qw/taskinst cupt cupt-r aptitude aptitude-r install unpack/;
%command = (
          "install" => "apt-get $qopt -d $ENV{aptoptions} -y --fix-missing install",
         "taskinst" => "aptitude -d $ENV{aptoptions} -y install $devnull",
              "apt" => "apt -d $ENV{aptoptions} -y install $devnull",
         "aptitude" => "aptitude -R -d $ENV{aptoptions} -y install $devnull",
       "aptitude-r" => "aptitude -r -d $ENV{aptoptions} -y install $devnull",
             "cupt" => "cupt -R --download-only $ENV{aptoptions} -y install $devnull",
           "cupt-r" => "cupt --download-only $ENV{aptoptions} -y install $devnull",
           "unpack" => "cd $downloaddir; aptitude download",
   "clean-internal" => 'true',
 "pending-internal" => 'true',
   "dpkgc-internal" => 'true',
);

}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub check_aptpkg {

  # check if the Perl module is available
  if (eval "require AptPkg::Config") {
    $use_aptpkg=1;
    eval "require AptPkg::System";
    eval "require AptPkg::Cache";

    $_config = $AptPkg::Config::_config;
    $_system = $AptPkg::System::_system;
    eval 'import AptPkg::Config $_config';
    eval 'import AptPkg::System $_system';
    eval 'import AptPkg::Cache';
  }
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -
sub showcommands {

  # show all available commands
  print "List of known commands for package_config files\n";
  print "Short list:\n";
  foreach (sort keys %command) {
    next if /-internal/; # skip internal commands
    print "$_\n";
  }

  print "\nLong list:\n";
  foreach (sort keys %command) {
    next if /-internal/; # skip internal commands
    printf "%15s    $command{$_}\n",$_;
  }
  exit 0;
}
# - - - - - - - - - - - - - - - - - - - - - - - - - - -