Merge pull request #1258 from jrha/spma-apt

ncm-spma: apt: Finish implementation

ncm-spma/src/main/perl/spma/apt.pm

@@ -1,5 +1,9 @@
 #${PMpre} NCM::Component::spma::apt${PMpost}
 
+use Data::Dumper;
+$Data::Dumper::Indent = 0; # Supress indentation and new-lines
+$Data::Dumper::Terse = 1; # Output values only, supress variable names if possible
+
 =head1 NAME
 
 C<NCM::Component::spma::apt> - NCM SPMA backend for apt
@@ -46,7 +50,7 @@ Packages listed under C</software/packages> will be installed, version and archi
 =cut
 
 use parent qw(NCM::Component CAF::Path);
-use CAF::Path 17.3.1;
+use CAF::Path 21.12.1;
 use CAF::Process;
 use CAF::FileWriter;
 use CAF::FileEditor;
@@ -67,23 +71,42 @@ Readonly my $TREE_SOURCES => "/software/repositories";
 Readonly my $TREE_PKGS => "/software/packages";
 Readonly my $BIN_APT_GET => "/usr/bin/apt-get";
 Readonly my $BIN_APT_MARK => "/usr/bin/apt-mark";
+Readonly my $BIN_APT_CACHE => "/usr/bin/apt-cache";
 Readonly my $BIN_DPKG_QUERY => "/usr/bin/dpkg-query";
 Readonly my $CMD_APT_UPDATE => [$BIN_APT_GET, qw(-qq update)];
 Readonly my $CMD_APT_UPGRADE => [$BIN_APT_GET, qw(-qq dist-upgrade)];
 Readonly my $CMD_APT_INSTALL => [$BIN_APT_GET, qw(-qq install)];
 Readonly my $CMD_APT_AUTOREMOVE => [$BIN_APT_GET, qw(-qq autoremove)];
 Readonly my $CMD_APT_MARK => [$BIN_APT_MARK, qw(-qq)];
+Readonly my $CMD_APT_AVAILABLE => [$BIN_APT_CACHE, qw(pkgnames)];
 Readonly my $CMD_DPKG_QUERY => [$BIN_DPKG_QUERY, qw(-W -f=${db:Status-Abbrev};${Package}\n)];
 
 our $NoActionSupported = 1;
 
+# Wrapper function for calling apt commands
+sub _call_apt
+{
+    my ($self, $cmd, $ok) = @_;
+    $self->debug(5, '_call_apt: Called with args ', Dumper($cmd));
+
+    my $proc = CAF::Process->new($cmd);
+    my $output = $proc->output();
+    my $exitstatus = $? >> 8; # Get exit status from highest 8-bits
+    $self->debug(5, "_call_apt: $proc exited with $exitstatus");
+    if ($exitstatus > 0) {
+        $output =~ tr{\n}{ };
+        my $method = $ok ? 'warn' : 'error';
+        $self->$method("_call_apt: $proc failed with \"$output\"");
+    }
+    return $ok || $exitstatus == 0;
+}
 
 # If user specified sources (userrepos) are not allowed, removes any
 # sources present in the system that are not listed in $allowed_sources.
 sub cleanup_old_sources
 {
     my ($self, $sources_dir, $allowed_sources) = @_;
-    $self->debug(5, 'Entered cleanup_old_sources()');
+    $self->debug(5, "cleanup_old_sources: Called with args ", $sources_dir, $allowed_sources);
 
     if ($self->directory_exists($sources_dir)) {
         my $current = Set::Scalar->new(@{$self->listdir($sources_dir, filter => qr{\.list$}, adddir => 1)});
@@ -108,7 +131,7 @@ sub cleanup_old_sources
 sub initialize_sources_dir
 {
     my ($self, $sources_dir) = @_;
-    $self->debug(5, 'Entered initialize_sources_dir()');
+    $self->debug(5, "initialize_sources_dir: Called with args($sources_dir)");
 
     if (! $self->directory($sources_dir)) {
         $self->error("Unable to create source dir $sources_dir: $self->{fail}");
@@ -124,7 +147,7 @@ sub initialize_sources_dir
 sub generate_sources
 {
     my ($self, $sources_dir, $sources, $template) = @_;
-    $self->debug(5, 'Entered generate_sources()');
+    $self->debug(5, "generate_sources: Called with args($sources_dir, $sources, $template)");
 
     my $changes = 0;
 
@@ -137,7 +160,7 @@ sub generate_sources
             $changes += $fh->close() || 0; # handle undef
         } else {
             $self->error("Invalid template '$template' passed to generate_sources");
-            return 0;
+            return;
         }
     }
 
@@ -148,26 +171,27 @@ sub generate_sources
 sub configure_apt
 {
     my ($self, $config) = @_;
-    $self->debug(5, 'Entered configure_apt()');
+    $self->debug(5, 'configure_apt: Called with args', Dumper($config));
 
     my $tr = EDG::WP4::CCM::TextRender->new($TEMPLATE_CONFIG, $config, relpath => 'spma');
     if ($tr) {
         my $fh = $tr->filewriter($FILE_CONFIG);
         return $fh->close() || 0; # handle undef
-    } else {
-        return 0;
     }
+    $self->error('configure_apt: TextRender failed to render configuration');
+    return;
 }
 
 # Returns a set of all installed packages
 sub get_installed_pkgs
 {
-    my $self = shift;
-    $self->debug(5, 'Entered get_installed_pkgs()');
+    my ($self) = @_;
+    $self->debug(5, 'get_installed_pkgs: Called');
 
     my $out = CAF::Process->new($CMD_DPKG_QUERY, keeps_state => 1) ->output();
-    if ($?) {
-        $self->debug(5, "dpkg command returned $?");
+    my $exitstatus = $? >> 8; # Get exit status from highest 8-bits
+    if ($exitstatus) {
+        $self->debug(5, "dpkg command returned $exitstatus");
         return 0;
     }
     # db:Status-Abbrev is three characters, we are looking for
@@ -179,34 +203,53 @@ sub get_installed_pkgs
     return Set::Scalar->new(@pkgs);
 }
 
+# Returns a set of all available package names
+sub get_available_pkgs
+{
+    my ($self) = @_;
+    $self->debug(5, 'get_available_pkgs: Called');
+
+    my $out = CAF::Process->new($CMD_APT_AVAILABLE, keeps_state => 1) ->output();
+    my $exitstatus = $? >> 8; # Get exit status from highest 8-bits
+    if ($exitstatus) {
+        $self->debug(5, "dpkg command returned $exitstatus");
+        return 0;
+    }
+    my @pkgs = split("\n", $out);
+
+    return Set::Scalar->new(@pkgs);
+}
+
 # For a given package name, extract version and architecture from tree passed in details
 # returns an arrayref of packages formatted with name, version and architecture for use with apt
 sub get_package_version_arch
 {
     my ($self, $name, $details) = @_;
-    $self->debug(5, 'Entered get_package_version_arch()');
+    $self->debug(5, "get_package_version_arch: Called with args($name, ", Dumper($details), ")");
 
     my @versions;
 
-    if ($details) {
+    if (defined($details) and %$details) {
         foreach my $version (sort keys %$details) {
             my $params = $details->{$version};
             $version = unescape($version);
             if ($params->{arch}) {
                 foreach my $arch (sort keys %{ $params->{arch} }) {
-                    $self->debug(5, '  Adding package ', $name, ' with version ', $version, ' and architecture ', $arch, ' to list');
+                    $self->debug(4, 'get_package_version_arch: Adding package ', $name, ' with version ', $version, ' and architecture ', $arch, ' to list');
                     push(@versions, sprintf('%s:%s=%s', $name, $arch, $version));
                 }
             } else {
-                $self->debug(5, '  Adding package ', $name, ' with version ', $version, ' but without architecture to list');
+                $self->debug(4, 'get_package_version_arch: Adding package ', $name, ' with version ', $version, ' but without architecture to list');
                 push(@versions, sprintf('%s=%s', $name, $version));
             }
         }
     } else {
-        $self->debug(5, '  Adding package ', $name, ' without version or architecture to list');
+        $self->debug(4, 'get_package_version_arch: Adding package ', $name, ' without version or architecture to list');
         push(@versions, $name);
     }
 
+    $self->debug(5, 'get_package_version_arch: returning arrayref:', Dumper(\@versions));
+
     return \@versions;
 }
 
@@ -216,7 +259,7 @@ sub apply_package_version_arch
 {
     my ($self, $packagelist, $packagetree) = @_;
 
-    $self->debug(5, 'Entered apply_package_version_arch()');
+    $self->debug(5, "apply_package_version_arch: Called with args", $packagelist, Dumper($packagetree));
 
     my @results;
     my @notfound;
@@ -240,7 +283,7 @@ sub apply_package_version_arch
 sub get_desired_pkgs
 {
     my ($self, $pkgs) = @_;
-    $self->debug(5, 'Entered get_desired_pkgs()');
+    $self->debug(5, "get_desired_pkgs: Called with args", Dumper($pkgs));
 
     my $packages = Set::Scalar->new();
 
@@ -259,33 +302,32 @@ sub get_desired_pkgs
 # Update package metadata from upstream sourcesitories
 sub resynchronize_package_index
 {
-    my $self = shift;
-    $self->debug(5, 'Entered resynchronize_package_index()');
+    my ($self) = @_;
+    $self->debug(5, 'resynchronize_package_index: Called');
 
-    my $cmd = CAF::Process->new($CMD_APT_UPDATE, keeps_state => 1);
-    return $cmd->execute() ? 1 : undef;
+    return $self->_call_apt($CMD_APT_UPDATE);
 }
 
 
 # Upgrade existing packages
 sub upgrade_packages
 {
     my ($self) = @_;
-    $self->debug(5, 'Entered upgrade_packages()');
+    $self->debug(5, 'upgrade_packages: Called');
 
-    my $cmd = CAF::Process->new($CMD_APT_UPGRADE) ;
-    return $cmd->execute() ? 1 : undef;
+    # it's ok if this produces errors (eg unfinished stuff)
+    # TODO: add support for 'apt --fix-broken install' and things like that
+    return $self->_call_apt($CMD_APT_UPGRADE, 1);
 }
 
 
 # Install packages
 sub install_packages
 {
     my ($self, $packages) = @_;
-    $self->debug(5, 'Entered install_packages()');
+    $self->debug(5, 'install_packages: Called with args', Dumper($packages));
 
-    my $cmd = CAF::Process->new([@$CMD_APT_INSTALL, @$packages]) ;
-    return $cmd->execute() ? 1 : undef;
+    return $self->_call_apt([@$CMD_APT_INSTALL, @$packages]);
 }
 
 
@@ -294,70 +336,83 @@ sub install_packages
 sub mark_packages_auto
 {
     my ($self, $packages) = @_;
-    $self->debug(5, 'Entered mark_packages_auto()');
+    $self->debug(5, "mark_packages_auto: Called with args", Dumper($packages));
 
-    my $cmd = CAF::Process->new([@$CMD_APT_MARK, 'auto', @$packages]) ;
-    return $cmd->execute() ? 1 : undef;
+    return $self->_call_apt([@$CMD_APT_MARK, 'auto', @$packages]);
 }
 
 
 # Remove automatically installed packages
 sub autoremove_packages
 {
     my ($self) = @_;
-    $self->debug(5, 'Entered autoremove_packages()');
+    $self->debug(5, 'autoremove_packages: Called');
 
-    my $cmd = CAF::Process->new([@$CMD_APT_AUTOREMOVE]) ;
-    return $cmd->execute() ? 1 : undef;
+    return $self->_call_apt([@$CMD_APT_AUTOREMOVE]);
 }
 
 
 sub Configure
 {
     my ($self, $config) = @_;
-    $self->debug(5, 'Entered Configure()');
 
     # Get configuration trees
     my $tree_sources = $config->getTree($TREE_SOURCES);
+    $self->debug(5, 'TREE_SOURCES ', $TREE_SOURCES, Dumper $tree_sources);
     my $tree_pkgs = $config->getTree($TREE_PKGS);
+    $self->debug(5, 'TREE_PKGS ', $TREE_PKGS, Dumper $tree_pkgs);
     my $tree_component = $config->getTree($self->prefix());
+    $self->debug(5, 'tree_component ', $self->prefix, Dumper $tree_component);
 
-    $self->configure_apt($tree_component) or return 0;
+    defined($self->configure_apt($tree_component)) or return 0;
 
-    $self->initialize_sources_dir($DIR_SOURCES) or return 0;
+    defined($self->initialize_sources_dir($DIR_SOURCES)) or return 0;
 
     # Remove unknown sources if allow_user_sources is not set
     if (! $tree_component->{usersources}) {
+        $self->info('Removing unknown source lists');
         $self->cleanup_old_sources($DIR_SOURCES, $tree_sources) or return 0;
     };
 
-    $self->generate_sources(
+    $self->info('Generating ', scalar(@$tree_sources), ' source lists');
+    defined($self->generate_sources(
         $DIR_SOURCES,
         $tree_sources,
         $TEMPLATE_SOURCES,
-    ) or return 0;
+    )) or return 0;
 
+    $self->info('Synchronizing package index');
     $self->resynchronize_package_index() or return 0;
 
+    $self->info('Applying upgrades to installed packages');
     $self->upgrade_packages() or return 0;
 
     my $packages_installed = $self->get_installed_pkgs() or return 0;
+    my $packages_available = $self->get_available_pkgs() or return 0;
     my $packages_desired = $self->get_desired_pkgs($tree_pkgs) or return 0;
-    my $packages_unwanted = $packages_installed->difference($packages_desired);
 
-    $self->debug(5, 'Installed packages:', $packages_installed);
-    $self->debug(5, 'Desired packages:', $packages_desired);
-    $self->debug(5, 'Packages installed but unwanted:', $packages_unwanted);
+    my $packages_unwanted = $packages_installed->difference($packages_desired);
+    my $packages_to_install = $packages_desired->difference($packages_installed);
+    my $packages_unavailable = $packages_desired->difference($packages_available);
 
-    my $packages_to_install = $self->apply_package_version_arch($packages_desired, $tree_pkgs) or return 0;
+    if ($packages_unavailable->size > 0) {
+        $self->warn('The following packages are unavailable, they may have been renamed or virtual: ', $packages_unavailable);
+    }
 
-    $self->debug(5, 'Packages to install ', $packages_to_install);
+    $self->debug(4, 'Installed packages: ', $packages_installed);
+    $self->debug(4, 'Desired packages: ', $packages_desired);
+    $self->debug(4, 'Unavailable packages: ', $packages_unavailable);
+    $self->debug(4, 'Packages installed but unwanted: ', $packages_unwanted);
+    $self->debug(4, 'Packages to install (desired but not installed): ', $packages_to_install);
 
-    $self->install_packages($packages_to_install) or return 0;
+    my $apt_packages_to_install = $self->apply_package_version_arch($packages_to_install, $tree_pkgs);
+    $self->info('Installing ', $packages_to_install->size,' missing packages');
+    $self->install_packages($apt_packages_to_install) or return 0;
 
     # If user installed packages are not permitted, mark all unlisted packages as automatically installed and
     # ask apt to remove any of these that are not required to satisfy dependencies of the desired package list
     if (! $tree_component->{userpkgs}) {
+        $self->info('Marking ', $packages_unwanted->size, ' packages as unwanted and removing any that are not dependencies of installed packages');
         $self->mark_packages_auto($packages_unwanted) or return 0;
         $self->autoremove_packages() or return 0;
     }

ncm-spma/src/main/resources/apt/source.tt

@@ -10,7 +10,7 @@
 # Name: [% name %]
 # Owner: [% owner %]
 
-[%- trust = gpgcheck ? "trusted=yes " : "" -%]
+[%- trust = gpgcheck ? "" : "[trusted=yes] " -%]
 [%- FOREACH source IN protocols %]
 # Protocol type: [% source.name %]
 deb [% trust %][% source.url %]