smt-ncc-sync

#!/usr/bin/perl

###############################################################################
## Copyright (c) 2007-2012 SUSE LINUX Products GmbH, Nuernberg, Germany.
###############################################################################

use strict;
use warnings;
use SMT::CLI;
use SMT::Utils;
use SMT::Mirror::RegData;
use SMT::NCCRegTools;
use File::Basename;
use File::Temp;

use Locale::gettext ();
use POSIX ();     # Needed for setlocale()

POSIX::setlocale(&POSIX::LC_MESSAGES, "");

#
# FIXME: what about /root/.curlrc for proxy settings?
#
if(!SMT::Utils::dropPrivileges())
{
    print STDERR __("Unable to drop privileges. Abort!\n");
    exit 1;
}

my $mirrorable;
my $domirror;

my $debug   = 0;
my $vblevel  = LOG_ERROR|LOG_WARN|LOG_INFO1;
my $logfile = "/dev/null";
my $help    = 0;
my $fromdir = "";
my $todir   = "";
my $exportdir = "";
my $createdbreplfile = "";
my $mail     = 0;
my $mailtempfile = undef;

my $exitcode = 0;

use Getopt::Long;
Getopt::Long::Configure( 'no_auto_abbrev');
my $optres = Getopt::Long::GetOptions(
                                      'fromdir=s'   => \$fromdir,
                                      'todir=s'     => \$todir,
                                      'createdbreplacementfile=s' => \$createdbreplfile,
                                      "logfile|L=s" => \$logfile,
                                      'debug|d'     => \$debug,
                                      "verboselevel|v=i" => \$vblevel,
                                      "mail|m"      => \$mail,
                                      "export=s"    => \$exportdir,
                                      'help|h'      => \$help
                                     );

if ( (defined $help && $help == 1 ) || !$optres )
{
    print basename($0) . " : " . __("Get data from NCC and update the local database.\n\n");
    print __("Usage:\n");
    print basename($0) . " [options]";
    print "\n\n";
    print __("Options:\n");
    print "--fromdir                 ".__("read NCC informations from directory instead of downloading it from NCC\n");
    print "                          ".__("and update the database.\n");
    print "--todir                   ".__("write NCC informations to directory without updating the database.\n");
    print "--createdbreplacementfile ".__("create a database replacement file for using smt-mirror without database.\n");
    print "--logfile -L file         ".__("Path to logfile\n");
    print "--debug                   ".__("Enable debug mode\n");
    print "--verboselevel -v level   ".__("set the verbose level\n");
    print "--export                  ".__("export NCC informations from the local database to a directory of XML files, the\n");
    print "                          ".__("resulting files can be imported on another SMT instance with --fromdir\n");
    print "--mail -m                 ".__("Send output as e-mail to the administrators defined in reportEmail in smt.conf.\n");
    print "                          ".__("The output on stdout and stderr will be suppressed.\n");
  exit 0;
}

$vblevel = LOG_ERROR|LOG_WARN|LOG_INFO1|LOG_INFO2|LOG_DEBUG|LOG_DEBUG2 if($debug);

if($mail)
{
    my $dir = File::Temp::tempdir("smt-XXXXXXXX", CLEANUP => 1, TMPDIR => 1);
    $mailtempfile = "$dir/mail";
    open(MAILTEMPFILE, "> $mailtempfile") or die "Cannot open file:$!";
    open(STDOUT, ">& MAILTEMPFILE") or die "Cannot dup:$!";
    open(STDERR, ">& MAILTEMPFILE") or die "Cannot dup:$!";
    select STDERR; $| = 1;      # make unbuffered
    select STDOUT; $| = 1;      # make unbuffered
    select MAILTEMPFILE; $| = 1;        # make unbuffered
}

if($createdbreplfile ne "")
{
    SMT::CLI::createDBReplacementFile($createdbreplfile);
    exit 0;
}

if($fromdir ne "" && $todir ne "")
{
    # both options set == no option set
    $fromdir = "";
    $todir   = "";
}

if(!SMT::Utils::openLock("smt-ncc-sync"))
{
    print __("Syncronization process is still running.\n");
    exit 0;
}

# open the logfile

my $LOG = SMT::Utils::openLog($logfile);

if($fromdir ne "" && $fromdir !~ /^\//)
{
    $fromdir = SMT::Utils::cleanPath($ENV{'PWD'}, $fromdir);
}

if($todir ne "" && $todir !~ /^\//)
{
    $todir = SMT::Utils::cleanPath($ENV{'PWD'}, $todir);
}

if($exportdir ne "" && $exportdir !~ /^\//)
{
    $exportdir = SMT::Utils::cleanPath($ENV{'PWD'}, $exportdir);
}

if($fromdir ne "" && ! -d $fromdir)
{
    SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, sprintf(__("Directory '%s' does not exist."), $fromdir));
    SMT::Utils::unLockAndExit( "smt-ncc-sync", 1, $LOG, $vblevel);
}

if( $exportdir ne "" )
{
    if ( $todir ne "" || $fromdir ne "" )
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR,
                __("--exportdir and --todir or --fromdir cannot be used at the same time."));
        SMT::Utils::unLockAndExit( "smt-ncc-sync", 1, $LOG, $vblevel);
    }
    if ( ! -d $exportdir )
    {
        eval {
                &File::Path::mkpath($exportdir);
        };
        if ($@)
        {
            SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, sprintf(__("Cannot create %s: %s"), $todir, $@));
            SMT::Utils::unLockAndExit( "smt-ncc-sync", 1, $LOG, $vblevel);
        }
    }
}

if($todir ne "" && ! -d $todir)
{
    # directory does not exists, try to create it.
    eval {
        &File::Path::mkpath($todir);
    };
    if ($@)
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, sprintf(__("Cannot create %s: %s"), $todir, $@));
        SMT::Utils::unLockAndExit( "smt-ncc-sync", 1, $LOG, $vblevel);
    }
}

if($fromdir ne "")
{
    printLog($LOG, $vblevel, LOG_INFO1, sprintf(__("Read from directory: %s"), $fromdir));
}
if($todir ne "")
{
    printLog($LOG, $vblevel, LOG_INFO1, sprintf(__("Write to directory: %s"), $todir));
}

if ( $exportdir ne "" )
{

    printLog($LOG, $vblevel, LOG_INFO1, __("Exporting repository information"));
    SMT::CLI::db2Xml(outfile => "$exportdir/catalogs.xml",
                     type => "catalogs",
                     table => "Catalogs",
                     columns => [ 'CATALOGID', 'NAME', 'DESCRIPTION', 'TARGET', 'EXTHOST', 'EXTURL',
                                  'CATALOGTYPE', 'DOMIRROR' ],
                     row_handler => \&catalog_handler
                    );

    printLog($LOG, $vblevel, LOG_INFO1, __("Exporting Product information"));
    SMT::CLI::db2Xml(outfile => "$exportdir/productdata.xml",
                     type => "productdata",
                     table => "Products",
                     columns => [ 'PRODUCTDATAID', 'PRODUCT', 'VERSION', 'REL', 'ARCH', 'FRIENDLY',
                                  'PARAMLIST', 'NEEDINFO', 'SERVICE', 'PRODUCT_LIST', 'PRODUCT_CLASS' ]
                    );

    printLog($LOG, $vblevel, LOG_INFO1, __("Exporting product/repository relations"));
    SMT::CLI::db2Xml(outfile => "$exportdir/productcatalogs.xml",
                     type => "productcatalogs",
                     table => "ProductCatalogs",
                     columns => [ 'PRODUCTDATAID', 'CATALOGID', 'OPTIONAL' ]
                    );

    printLog($LOG, $vblevel, LOG_INFO1, __("Exporting Target information"));
    SMT::CLI::db2Xml(outfile => "$exportdir/targets.xml",
                     type => "targets",
                     table => "Targets",
                     columns => [ 'OS', 'TARGET' ]
                    );

    printLog($LOG, $vblevel, LOG_INFO1, __("Exporting Subscription information"));
    SMT::CLI::db2Xml(outfile => "$exportdir/listsubscriptions.xml",
                     type => "subscriptionlist",
                     table => "Subscriptions",
                     columns => [ "SUBID", "REGCODE", "SUBNAME", "SUBTYPE", "SUBSTATUS",
                                  "SUBSTARTDATE", "SUBENDDATE", "SUBDURATION", "SERVERCLASS",
                                  "PRODUCT_CLASS" , "NODECOUNT", "CONSUMED", "CONSUMEDVIRT"],
                     row_handler => \&subscriptions_handler );

    # an empty registrations file should be enough for our purposes (later import into
    # an isolated SMT)
    my $output = new IO::File("> $exportdir/listregistrations.xml");
    if(!defined $output)
    {
        die "Cannot open file '$exportdir/listregistrations.xml':$!";
    }
    my $writer = new XML::Writer(OUTPUT => $output);
    $writer->xmlDecl("UTF-8");
    $writer->startTag("registration", xmlns => "http://www.novell.com/xml/center/regsvc-1_0");
    $writer->endTag("registration");
    $writer->end();
    $output->close();

    SMT::Utils::unLockAndExit( "smt-ncc-sync", 0, $LOG, $vblevel);
}

eval
{
    my $rd= SMT::Mirror::RegData->new(vblevel => $vblevel,
                                      log     => $LOG,
                                      element => "productdata",
                                      table   => "Products",
                                      key     => "PRODUCTDATAID",
                                      fromdir => (($fromdir ne "")?$fromdir:undef),
                                      todir   => (($todir ne "")?$todir:undef));
    printLog($LOG, $vblevel, LOG_INFO1, __("Downloading Product information"));

    my $res = $rd->sync();
    if ($res)
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, __("Error while fetching Products data."));
        $exitcode = 1;
    }


    $rd->element("targets");
    $rd->table("Targets");
    $rd->key("OS");

    printLog($LOG, $vblevel, LOG_INFO1, __("Downloading Target information"));
    $res = $rd->sync();
    if ($res)
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, __("Error while fetching Targets data."));
        $exitcode = 1;
    }

    $rd->element("catalogs");
    $rd->table("Catalogs");
    $rd->key("CATALOGID");

    printLog($LOG, $vblevel, LOG_INFO1, __("Downloading repository information"));
    $res = $rd->sync();
    if ($res)
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, __("Error while fetching repository data."));
        $exitcode = 1;
    }

    $rd->element("productcatalogs");
    $rd->table("ProductCatalogs");
    $rd->key(['PRODUCTDATAID', 'CATALOGID']);

    printLog($LOG, $vblevel, LOG_INFO1, __("Downloading Product/Repository relations"));
    $res = $rd->sync();
    if ($res)
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, __("Error while fetching ProductCatalogs data."));
        $exitcode = 1;
    }


    my $lr= SMT::NCCRegTools->new(vblevel => $vblevel,
                                  log     => $LOG,
                                  fromdir => (($fromdir ne "")?$fromdir:undef),
                                  todir   => (($todir ne "")?$todir:undef));

    printLog($LOG, $vblevel, LOG_INFO1, __("Downloading Subscription information"));
    $res = $lr->NCCListSubscriptions();
    if ($res)
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, __("Error while fetching Subscription data."));
        $exitcode = 1;
    }

    printLog($LOG, $vblevel, LOG_INFO1, __("Downloading Registration information"));
    $res = $lr->NCCListRegistrations();
    if ($res)
    {
        SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, __("Error while fetching Registration data."));
        $exitcode = 1;
    }

    if ($todir ne "")
    {
        SMT::CLI::setMirrorableCatalogs(todir   => $todir,
                                        log     => $LOG,
                                        vblevel => $vblevel);
        # if todir parameter is set, we can stop here. Everything is done.
        SMT::Utils::unLockAndExit("smt-ncc-sync", 0, $LOG, $vblevel);
    }

    printLog($LOG, $vblevel, LOG_INFO1, __("Flagged repositories which can be mirrored"));
    SMT::CLI::setMirrorableCatalogs(fromdir => ($fromdir ne "")?$fromdir:undef,
                                    todir   => ($todir ne "")?$todir:undef,
                                    log     => $LOG,
                                    vblevel => $vblevel);
    if( $fromdir ne "" && -d $fromdir )
    {
        my $mirrorinfofile = $fromdir."/mirrorinfo.xml";
        if ( -f $mirrorinfofile && (stat($fromdir."/catalogs.xml"))[9] <= (stat($mirrorinfofile))[9] )
        {
            SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, __("Updating mirror flags from $mirrorinfofile."));
            SMT::CLI::setDoMirrorFromXml( xml => $mirrorinfofile );
        }
    }

};
if ($@)
{
    SMT::Utils::printLog($LOG, $vblevel, LOG_ERROR, $@);
    $exitcode = 1;
}

if($mail)
{
    close (STDERR);
    close (STDOUT);
    close (MAILTEMPFILE);
    my $body = "";

    open(MAIL, "< $mailtempfile") and do
    {
        while(<MAIL>)
        {
            $body .= $_;
        }
        close MAIL;
    };
    my $datestring = POSIX::strftime("%Y-%m-%d %H:%M", localtime);
    my $subject = sprintf("SMT Sync Report $datestring (%s) -- Sync %s",
                          SMT::Utils::getFQDN(), (($exitcode > 0)?"failed":"successful"));

    SMT::Utils::sendMailToAdmins($subject, $body);
}

SMT::Utils::unLockAndExit("smt-ncc-sync", $exitcode, $LOG, $vblevel);


sub subscriptions_handler
{
    my ($writer,$row) = @_;
    # The SQL column name don't map 1to1 to the XML tag names for the Subscriptions-Table
    my %col2tag = (
        "SUBID"         => "subid",
        "REGCODE"       => "regcode",
        "SUBNAME"       => "subname",
        "SUBTYPE"       => "type",
        "SUBSTATUS"     => "substatus",
        "SUBSTARTDATE"  => "start-date",
        "SUBENDDATE"    => "end-date",
        "SUBDURATION"   => "duration",
        "SERVERCLASS"   => "server-class",
        "PRODUCT_CLASS" => "product-class",
        "NODECOUNT"     => "nodecount",
        "CONSUMED"      => "consumed",
        "CONSUMEDVIRT"  => "consumed-virtual"
    );
    $writer->startTag("subscription");
    foreach my $col (keys %{$row})
    {
        $writer->startTag($col2tag{$col});
        $writer->characters( $row->{$col} ?  $row->{$col} : "0");
        $writer->endTag($col2tag{$col});
    }
    $writer->endTag("subscription");
}

sub catalog_handler
{
    my ($writer,$row) = @_;
    $writer->startTag("row");
    foreach my $col (keys %{$row})
    {
        $writer->startTag("col", name => ( $col eq "DOMIRROR" ? "MIRRORABLE" : $col ));
        $writer->characters($row->{$col});
        $writer->endTag("col");
    }
    $writer->endTag("row");
}

#
# Manpage
#

=head1 NAME

smt ncc-sync -

=head1 SYNOPSIS

smt ncc-sync [options]

=head1 DESCRIPTION

C<smt ncc-sync> gets data from Novell Customer Center and updates the local database.

=head1 OPTIONS

=head2 COMMANDLINE

=over

=item --fromdir directory

Reads NCC information from directory instead of downloading it from NCC and update the database.

=item --todir directory

Write NCC informations to directory without updating the database.

=item --createdbreplacementfile

Creates a database replacement file for using smt-mirror without database.

=item --logfile -L file

Path to logfile.

=item --debug

Enable display of debug information.

=item --verboselevel -v <level>

Set the output verbose level. The following categories exists.
These categories can be bitwise-or'd to use as verbose level.

=over 4

=item error messages

Value: 0x0001 == 1

=item warning messages

Value: 0x0002 == 2

=item info messages 1

Value: 0x0004 == 4

=item info messages 2

Value: 0x0008 == 8

=item debug messages 1

Value: 0x0010 == 16

=item debug messages 2

Value: 0x0020 == 32

=item debug messages 3

Value: 0x0040 == 64

=back

The default verbose level is 15 (error, warning and all info messages).
B<--debug> set the verbose level to 63.

=item --mail -m

Send output as e-mail to the administrators defined in reportEmail in smt.conf .
The output on stdout and stderr will be suppressed in this mode.

=item --export directory

Export the NCC informations from the local database to a directory of XML files,
that directory can later be imported on another SMT instance with
C<smt ncc-sync --fromdir directory>.

=back

=head1 AUTHORS and CONTRIBUTORS

Duncan Mac-Vicar Prett, Lukas Ocilka, Jens Daniel Schmidt, Michael Calmer

=head1 LICENSE

Copyright (c) 2007-2012 SUSE LINUX Products GmbH, Nuernberg, Germany.

This program is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free Software
Foundation; either version 2 of the License, or (at your option) any later
version.

This program is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
PARTICULAR PURPOSE. See the GNU General Public License for more details.

You should have received a copy of the GNU General Public License along with
this program; if not, write to the Free Software Foundation, Inc., 675 Mass
Ave, Cambridge, MA 02139, USA.

=cut