Skip to content
Permalink
master
Switch branches/tags
Go to file
 
 
Cannot retrieve contributors at this time
#!/usr/bin/perl
#emacs: -*- mode: shell-script; c-basic-offset: 4; tab-width: 4; indent-tabs-mode: t -*-
#ex: set sts=4 ts=4 sw=4 noet:
#
require 5.002;
use Socket;
my $INFO = <<'END_INFO';
Usage: nd_freeze [--keep-apt-sources|-k] [--no-updates|-n] [--debug|-d] [--help|-h] [--trust-repos|-t] date
Install access to the NeuroDebian snapshot repository that allows installation
of packages based on a given date.
-k, --keep-apt-sources Leave the original, non-snapshot source active so
that originally sourced packages can still be
installed.
-n, --no-updates Make the snapshot repositories available but skip
the package update step, leaving the system as is.
-d, --debug Display debugging info to the screen.
-h, --help Display this help text and exit.
-t, --trust-repos Update untrusted repos. For repos whose apt keys
are invalid, go ahead and use the repo rather than
failing by default. USE CAUTIOUSLY!
The UTC date/time format sent to NeuroDebian to get the repo info is in the
format yyyymmddThhmmssz or yyymmdd. However, the script will handle and
reformat a number of date/time formats. Quotes are required if there is a
space between the date and time components:
yyyymmddThh:mm:ssZ
yyyymmdd
mm/dd/yyyy
yyyy-mm-dd
yyyymmddThh:mm
"yyyy-mm-dd hh:mm:ss"
"mm/dd/yy hh:mm:ss"
Browse available snapshots here:
http://snapshot-neuro.debian.net/archive/neurodebian/
If there is no snapshot available at the exact time you specified you will get
both the previous and next available timestamped repos of the one you specified
to make available versions that may have been placed in the repo before your
selected time but after the previous snapshot.
END_INFO
# Function prints information to the screen. "logg" with 2 g's because there
# is a keyword 'log' (the math function)
#
# Parameters
# ----------
# $label
# string : valid values: 'info', 'debug'
# debug values are only displayed when --debug switch set
# $output
# string : Text to print to the screen.
#
sub logg {
my $label = shift(@_);
return if (!$DEBUG and $label eq 'debug');
$label = uc($label);
foreach my $line (@_) {
print "${label}: ${line}";
print "\n" if (!($line =~ /\n$/));
}
}
# Function gets the page contents of the provided URL.
#
# Parameters
# ----------
# $url
# string : URL of page to retrieve
#
# Returns
# -------
# string : Contents of URL page
#
sub get_www_content {
my ($url) = @_;
$url =~ /http:\/\/([^\/:]+):?(\d+)*(.*)/;
my $dest = $1;
my $port = $2;
my $file = $3;
$port = 80 if (!$port);
my $proto = getprotobyname('tcp');
socket(F, PF_INET, SOCK_STREAM, $proto);
my $sin = sockaddr_in($port,inet_aton($dest));
connect(F, $sin) || return undef;
my $old_fh = select(F);
$| = 1;
select($old_fh);
print F "GET $file HTTP/1.1\r\nHost: ${dest}\r\n\r\n";
$/ = undef;
$contents = <F>;
close(F);
return $contents;
}
# Run a shell command
#
# Parameters
# ----------
# $command
# string : command to run in shell
#
sub exec_shell {
my $command = shift;
logg("debug", "Executing shell command: '$command'");
my $stdout = qx!$command 2>&1!;
if ($?) {
logg("info", "Command '$command' failed.");
exit $?;
}
logg("debug", split(/\n/, $stdout));
return $stdout;
}
# Function to run apt-get update
sub run_apt_get_update {
logg("info", "Refreshing apt cache");
my $switch = "";
$switch = "--no-allow-insecure-repositories" if (is_minimum_apt_version("1.4.8"));
my $stdout = exec_shell("apt-get update $switch");
if ($stdout =~ /NO_PUBKEY (\w+)/) {
logg("info", "Missing public key for $1, attempting to get key");
exec_shell("apt-key adv --recv-keys --keyserver hkps://keyserver.ubuntu.com 0x${1}");
logg("info", "Successfully retrieved public key, re-running apt update");
exec_shell("apt-get update $switch");
}
}
# Function validates and then returns the user provided freeze date in the
# format yyyymmddThh:mm:ssZ.
#
# Returns
# -------
# string : User date in format yyyymmddThh:mm:ssZ
#
sub get_user_timestamp {
my ($user_date) = @_;
# Check for date command line argument
if (!$user_date) {
logg("info", '
Script to enable a NeurDebian archived snapshot repository. Snapshot repositories
provide previous releases of packages based on point-of-time archiving.
See:
https://snapshot.debian.org/
http://snapshot-neuro.debian.net/
Synopsis
--------
nd_freeze <date>
Valid date formats include:
yyyymmddThh:mm:ssZ
yyyymmdd
mm/dd/yyyy
yyyy-mm-dd
yyyymmddThh:mm
"yyyy-mm-dd hh:mm:ss"
"mm/dd/yy hh:mm:ss"
');
exit 1;
}
# Prepend any single date or time values with a 0
my @chunks = split /[\/\- TZ\:]/, $user_date;
for my $i (0 .. $#chunks) {
if (length $chunks[$i] == 1) {
$chunks[$i] = "0$chunks[$i]";
}
}
# Pull the date from the parameter string
my $url_date = '';
if (length $chunks[0] == 8) {
$url_date = $chunks[0];
} elsif (length $chunks[0] == 4) {
$url_date = "$chunks[0]$chunks[1]$chunks[2]";
} elsif (length $chunks[0] == 2) {
$url_date = "$chunks[2]$chunks[0]$chunks[1]";
}
# Pull the time from the parameter string
my $url_time = '';
if ((scalar @chunks >= 2) && (length $chunks[-1] == 6)) {
$url_time = "$chunks[-1]";
} elsif ((length $chunks[0] > 4 && scalar @chunks == 3) || scalar @chunks == 5) {
$url_time = "$chunks[-2]$chunks[-1]00";
} elsif ((length $chunks[0] > 4 && scalar @chunks == 4) || scalar @chunks == 6) {
$url_time = "$chunks[-3]$chunks[-2]$chunks[-1]";
}
if (!$url_date) {
logg("info", '
ERROR: Invalid date
Valid date formats include:
yyyymmddThh:mm:ssZ
yyyymmdd
mm/dd/yyyy
yyyy-mm-dd
yyyymmddThh:mm
"yyyy-mm-dd hh:mm:ss"
"mm/dd/yy hh:mm:ss"
');
exit 1;
}
$user_timestamp='';
if ($url_time) {
$user_timestamp = "${url_date}T${url_time}Z";
} else {
$user_timestamp = "${url_date}T000000Z";
}
return $user_timestamp;
}
# Function returns a hash of information for each source that we want
# to replace with a snapshot.
#
# Returns
# -------
# %sources
# hash : The sources retrieved from apt-cache policy that need to be written to the sources file.
#
sub get_sources {
my %sources;
logg("info", "Discovering installed repository sources");
run_apt_get_update();
my @files = glob("/var/lib/apt/lists/*Release");
for my $file (@files) {
my %source;
open my $fh, '<', $file or die "Could not open '$file' ${!}\n";
while (my $line = <$fh>) {
chomp $line;
if ($line =~ /^([a-zA-Z][^:]*):[\ ]+(\S.*)$/) {
$source{$1} = $2;
}
}
if ($source{Origin} eq "Debian" or $source{Origin} eq "NeuroDebian" or $source{Origin} eq "Debian Backports") {
logg("debug", "Found $source{Label} $source{Codename} with components $source{Components}");
for my $component (split(' ', $source{Components})) {
my $codename = $source{Codename};
if ($component =~ /(\S+)\/(\S+)/) {
$codename = "${source{Codename}}/${1}";
$component = $2;
}
my $key = "$source{Label}|$source{Codename}|$component";
%{$sources{$key}} = (
Origin => $source{Origin},
Label => $source{Label},
Codename => $codename,
Component => $component
);
}
}
}
return %sources;
}
# Function writes out the necessary lines to the /etc/apt/sources.list.d/snapshot.sources.list
# file. The source written to the sources file is pointed to the next snapshot
# taken after the date provided by the user. To get the "next" snapshot timestamp
# we pull the HTML file of the current snapshot and scrape the next timestamp.
#
# Parameters
# ----------
# $snapshots_sources_file
# string : Path to snapshot sources file.
# $user_timestamp
# string : Timestamp of freeze date provided by user at the command line.
# %sources
# hash : The sources retrieved from /var/lib/apt/lists/*Release policy that
# need to be written to the sources file.
#
sub write_snapshot_sources {
my ($snapshots_sources_file, $user_timestamp, %sources) = @_;
my $found_count = 0;
open my $fp, '>', $snapshots_sources_file;
for my $key (keys %sources) {
my $domain = 'snapshot-neuro.debian.net';
$domain = 'snapshot.debian.org' if ($sources{$key}{Origin} eq 'Debian' || $sources{$key}{Origin} eq 'Debian Backports');
my $label = lc($sources{$key}{Label});
$label =~ tr/ /-/;
$contents = get_www_content("http://${domain}/archive/${label}/${user_timestamp}/");
# Handle bad URL requests
if (!($contents =~ /^HTTP\/\d+\.\d+ 200/) and !($contents =~ /^HTTP\/\d+\.\d+ 301/)) {
$contents =~ /^(.+)/;
die "Bad URL request http://${domain}/archive/${label}/${user_timestamp}/: ${1}\n";
}
# Handle 301 redirect from snapshot server if we get one.
if ($contents =~ /The resource has been moved to http:\/\/[\S\-]+\/archive\/${label}\/(\d{8}T\d{6}Z\/)/) {
$contents = get_www_content("http://${domain}/archive/${label}/${1}/");
}
# Scrape next timestamp from HTML returned from snapshot server.
$contents =~ /\/archive\/${label}\/([0-9TZ]+)\/">next</;
my $next_timestamp = $1;
$next_timestamp =~ tr/\///d;
if ($user_timestamp gt $next_timestamp) {
# Notify user that they have requested a date beyond the most recent snapshot.
$last_available = $next_timestamp ? $next_timestamp : $user_timestamp;
logg("info", "ERROR: User specified time (${user_timestamp}) must predate the most recent snapshot timestamp (${last_available}). Freeze failed.");
close $fp;
unlink $snapshots_sources_file or warn "Could not delete $snapshots_sources_file: $!";
exit(1);
}
# Hold off updating security archive until after the debian-archive-keyring package is updated.
my $line_prefix = "";
$line_prefix = "#" if ($sources{$key}{Label} eq 'Debian-Security');
my $trusted_option = ( ($TRUST_REPOS and is_minimum_apt_version("0.9.0")) ? " [trusted=yes]" : "" );
my $source_line = "deb$trusted_option http://${domain}/archive/${label}/${next_timestamp}/ ${sources{$key}{Codename}} ${sources{$key}{Component}}";
print $fp "${line_prefix}${source_line}\n";
logg("debug", "Adding '${source_line}' to $snapshots_sources_file");
$found_count += 1;
}
close $fp;
return $found_count;
}
# Function comments out the lines in the debian and neurodebian sources files
# that we are replacing with our snapshot sources.
#
# Parameters
# ----------
# $sources_file
# string : Path to sources file to update.
# %sources
# hash : The sources retrieved from apt-cache policy that need to be written to the sources file.
#
sub disable_lines {
my ($sources_file, %sources) = @_;
exec_shell("cp $sources_file ${sources_file}.orig.disabled") if (!-e "${sources_file}.orig.disabled");
open my $in, '<', "${sources_file}.orig.disabled"
or die "Could not open file '${sources_file}.orig.disabled' ${!}\n";
open my $out, '>', $sources_file
or die "Could not open file '$sources_file' ${!}\n";
my @lines = split /\n/, <$in>;
foreach (@lines) {
# Skip commented lines
if (/^#/) {
print $out "$_\n";
next;
}
my $found = 0;
# Loop through the sources from the Release files for each line in the
# sources file to determine if it is one we need to comment out.
my $file_line = '';
my $source_line = '';
for my $key (keys %sources) {
$_ =~ /(http:\S+)(.*)$/;
$file_line = $1 . join ' ', sort split /\s+/, $2;
my $url;
if ($sources{$key}{Label} eq "Debian" || $sources{$key}{Label} eq "Debian Backports") {
$url = "http://deb.debian.org/debian";
}
elsif ($sources{$key}{Label} eq "Debian-Security") {
$url = "http://security.debian.org/debian-security";
}
elsif ($sources{$key}{Label} eq "NeuroDebian") {
$url = "http://neuro.debian.net/debian";
}
$source_line = $url . ' ' . join ' ', sort split /\s+/, "$sources{$key}{Codename} $sources{$key}{Component}";
if ($file_line eq $source_line) {
$found = 1;
last;
}
}
if ($found) {
print $out "# $_\n";
logg("debug", "Disabling '$source_line' in $sources_file");
} else {
print $out "$_\n";
}
}
close $in;
close $out;
}
# Get a list of the installed packages
#
# Returns
# ----------
# @packages
# array : Names of the installed packages
#
sub installed_packages {
logg("info", "Discovering installed packages for possible version updates");
my @packages = ();
my @lines = split /\n/, exec_shell("dpkg -l");
for my $i (0 .. $#lines) {
my @chunks = split /\s+/, $lines[$i];
push @packages, $chunks[1] if ($chunks[0] eq 'ii');
}
return @packages;
}
# Update the installed packages to their snapshot versions
#
# Parameters
# ----------
# @packages
# array : List of installed packages
#
sub update_packages {
my (@packages) = @_;
my @update_packages;
for my $package (@packages) {
my $found = 0;
my $installed_version = '';
my $snapshot_version = '';
my @lines = split /\n/, exec_shell("apt-cache policy $package");
for my $i (0 .. $#lines) {
$installed_version = $1 if ($lines[$i] =~ /Installed:\s+(\S+)/);
if ($lines[$i] =~ /snapshot\.debian\.org|snapshot\-neuro\.debian\.net/) {
my @chunks = split /\s+/, $lines[$i-1];
$snapshot_version = $chunks[1];
$found = 1 if ($snapshot_version ne '***');
last;
}
}
if ($found) {
my $version_test = exec_shell("dpkg --compare-versions ${snapshot_version} ge ${installed_version} && echo 1 || echo 0");
my $action = 'DOWNGRADE';
$action = 'UPGRADE' if ($version_test == 1);
logg("debug", "Will ${action} '${package}=${installed_version}' to version '${snapshot_version}'");
push @update_packages, "${package}=${snapshot_version}";
}
}
if (scalar(@update_packages) > 0) {
my $packages_to_update = join " ", @update_packages;
logg("info", "UPDATING: ${packages_to_update}");
my $downgrade_switch = is_minimum_apt_version("1.1.57") ? "--allow-downgrades" : "";
my $remove_switch = is_minimum_apt_version("1.1") ? "--allow-remove-essential" : "";
my $force_switch = ($downgrade_switch || $remove_switch) ? "" : "--force-yes";
exec_shell("apt install --fix-broken $force_switch $downgrade_switch $remove_switch -y");
exec_shell("export DEBIAN_FRONTEND=noninteractive; apt-get install --no-install-recommends $force_switch $downgrade_switch $remove_switch -y ${packages_to_update}");
}
}
# Test system for minimum version of apt package
#
# Parameters
# ----------
# $version
# string : version number to test
#
# Returns
# -------
# boolean : true if system version is >= to the parameter version
#
sub is_minimum_apt_version {
my $version = shift;
my $result = exec_shell("dpkg -l apt");
$result =~ /ii\s+apt\s+(\S+)/;
logg("debug", "Found apt version $1");
my $version_test = exec_shell("dpkg --compare-versions $1 ge $version && echo 1 || echo 0");
$version_test =~ /(\d+)/;
return $1;
}
##### Program main
# Global vars
$USER_DATE = "";
$KEEP_SOURCES = 0;
$DO_NOT_UPDATE = 0;
$DEBUG = 0;
$HELP = 0;
$TRUST_REPOS = 0;
# Parse command line options
foreach(@ARGV) {
$KEEP_SOURCES = 1 if ($_ eq '--keep-apt-sources' or $_ eq '-k');
$DO_NOT_UPDATE = 1 if ($_ eq '--no-updates' or $_ eq '-n');
$DEBUG = 1 if ($_ eq '--debug' or $_ eq '-d');
$HELP = 1 if ($_ eq '--help' or $_ eq '-h');
$TRUST_REPOS = 1 if ($_ eq '--trust-repos' or $_ eq '-t');
}
my $parameter_count = 1 + $KEEP_SOURCES + $DO_NOT_UPDATE + $DEBUG + $TRUST_REPOS;
if (scalar(@ARGV) != $parameter_count) {
print "Invalid command parameters\n";
print "USAGE: nd_freeze [--keep-apt-sources] [--no-updates] [--debug] [--help] date\n";
exit 1;
}
if ($HELP) {
print $INFO;
exit(0);
}
$USER_DATE = $ARGV[$parameter_count - 1];
# Set apt setting to allow "outdated" repositories.
if (! -e '/etc/apt/apt.conf.d/10no--check-valid-until') {
exec_shell("echo 'Acquire::Check-Valid-Until \"0\";' > /etc/apt/apt.conf.d/10no--check-valid-until");
}
# So we later on could make a decision either we need to clean up after ourselves
# Rely on having a Release file, since cannot be just * since there could be
# empty partial/ directory
my @apt_releases_list = </var/lib/apt/lists/*Release>;
my $user_timestamp = get_user_timestamp($USER_DATE);
my $snapshots_sources_file = '/etc/apt/sources.list.d/snapshots.sources.list';
my @sources_files = (
'/etc/apt/sources.list',
'/etc/apt/sources.list.d/neurodebian.sources.list',
'/etc/apt/sources.list.d/backports.list'
);
# Restore original sources files and apt-cache if this is a rerun of the command.
foreach my $sources_file (@sources_files) {
exec_shell("cp ${sources_file}.orig.disabled $sources_file") if (-e "${sources_file}.orig.disabled");
}
if (-e $snapshots_sources_file) {
exec_shell("rm $snapshots_sources_file");
run_apt_get_update();
}
my %sources = get_sources();
if ((keys %sources) > 0) {
my $found_count = write_snapshot_sources($snapshots_sources_file, $user_timestamp, %sources);
if ($found_count > 0 and !$KEEP_SOURCES) {
foreach my $sources_file (@sources_files) {
disable_lines($sources_file, %sources) if (-e $sources_file);
}
run_apt_get_update(); # run even only to check and then cleanup later on
}
} else {
logg("info", "No valid sources found to get snapshots.");
exit 1;
}
if ($DO_NOT_UPDATE) {
logg("info", "Packages were not updated");
# Enable security archives
exec_shell("sed -i 's:#::g' $snapshots_sources_file");
}
else {
# Update debian-archive-keyring first
update_packages(("debian-archive-keyring"));
# Enable security archives
exec_shell("sed -i 's:#::g' $snapshots_sources_file");
run_apt_get_update();
# Update all the packages
my @packages = installed_packages();
update_packages(@packages);
}
if (scalar @apt_releases_list == 0) {
logg("info", "Cleaning up APT lists because originally there were none");
exec_shell("rm -rf /var/lib/apt/lists/*");
}
exit 0