mk/bulk/post-build

#!/usr/pkg/bin/perl
# $NetBSD: post-build,v 1.45 2004/07/15 11:57:26 grant Exp $
#
# Collect stuff after a pkg bulk build
#
# (c) 2000 Hubert Feyrer, All Rights Reserved.
#

use File::Basename;
use POSIX ();

# Where config vars are stored (/bin/sh syntax)

if (-f $ENV{"BULK_BUILD_CONF"}) {
	$BULK_BUILD_CONF=$ENV{"BULK_BUILD_CONF"};
} else {
	$BULK_BUILD_CONF=dirname("$0")."/build.conf";
}

# Dig given variable out of config file, and set it
sub getconf {
	local($var)=@_;
	local($val);

	if ($BULK_BUILD_CONF =~ /^\//) {
		chomp($val=`. $BULK_BUILD_CONF ; echo \$$var`);
	} else {
		chomp($val=`. ./$BULK_BUILD_CONF ; echo \$$var`);
	}
	eval "\$$var=\"$val\";";
}

getconf("ADMINSIG");		# "-Your Name"
getconf("FTPURL");		# "pub/NetBSD/pkgstat/`date +%Y%m%d.%H%M`"
getconf("FTP");			# "/disk1/ftp/${FTPURL}"
getconf("FTPHOST");		# ftp://ftp.machi.ne/
getconf("REPORT");		# "broken.html"
getconf("USR_PKGSRC");		# "/usr/pkgsrc"
getconf("osrev");	       	# `uname -r`
getconf("arch"); 		# `uname -m`
$reportf=basename($REPORT);

chomp($os=`uname -s`);

# Extract the name of the files used for the build log and broken build log.
# These have defaults set by bsd.bulk-pkg.mk and may be overridden in
# /etc/mk.conf
chomp($BROKENFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=BROKENFILE )`);

# Also extract the names of the cache files used during the bulk build.  We'll
# save a copy of those to help debug if the build messed up.
chomp($BULK_DBFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=BULK_DBFILE )`);
chomp($DEPENDSTREEFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=DEPENDSTREEFILE )`);
chomp($DEPENDSFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=DEPENDSFILE )`);
chomp($SUPPORTSFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=SUPPORTSFILE )`);
chomp($INDEXFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=INDEXFILE )`);
chomp($ORDERFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=ORDERFILE )`);
chomp($STARTFILE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=STARTFILE )`);
chomp($LOCALBASE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=LOCALBASE )`);
chomp($X11BASE=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=X11BASE )`);
chomp($FIND=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=FIND )`);
chomp($GTAR=`( cd $USR_PKGSRC/pkgtools/pkglint ; $ENV{"BMAKE"} show-var VARNAME=GTAR )`);
$bulkdbfile=basename($BULK_DBFILE);
$dtfile=basename($DEPENDSTREEFILE);
$depfile=basename($DEPENDSFILE);
$supfile=basename($SUPPORTSFILE);
$indfile=basename($INDEXFILE);
$ordfile=basename($ORDERFILE);
$startfile=basename($STARTFILE);

$startdate = (stat($STARTFILE))[9];
if ($startdate == 0) {
	$startdate = "unknown";
} else {
	local $ENV{TZ} = "UTC";
	$startdate = POSIX::strftime("%c %Z", gmtime($startdate));
	$enddate = POSIX::strftime("%c %Z", gmtime(time()));
}

$verbose=1;

$report_head = <<EOF;
Package                    Breaks  Maintainer
--------------------------------------------------------------
EOF
$report_form = <<EOF;
@<<<<<<<<<<<<<<<<<<<<<<<<< @<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<
$pkg                       $nbrokenby  $maintainer
EOF

system("mkdir -p ${FTP}");

# Copy over the output from the build process
chdir($USR_PKGSRC);
system("find . -name $BROKENFILE -print | $GTAR -T - -plcf - | (cd $FTP; $GTAR -plxf -)");

# Copy over the cache files used during the build
foreach my $f ($BULK_DBFILE, $DEPENDSTREEFILE, $DEPENDSFILE, $SUPPORTSFILE, $INDEXFILE, $ORDERFILE) {
	system("cp ${f} ${FTP}") if -f ${f};
}

chdir($FTP);
writeReport();

#
# Adjust "last" symlink
#
{
	local($base, $dir) = $FTP=~m|^(.*)/([^/]*)$|;

	unlink("$base/last");
	symlink($dir, "$base/last");
}

#
# Generate leftovers-${arch}.html: files not deleted
# Leftover files are copied to leftovers-${arch} dir, and linked from
# leftovers-${arch}.html
#
{
	chdir($FTP);
	system("mkdir -p leftovers-${arch}");

	# Find files since last build:
	$leftovers_txt = "leftovers-${arch}.txt";
	$leftovers_html = "leftovers-${arch}.html";
	system("${FIND} ${LOCALBASE} -newer ${USR_PKGSRC}/${startfile} -type f -print >>$leftovers_txt");
	system("${FIND} ${X11BASE} -newer ${USR_PKGSRC}/${startfile} -type f -print >>$leftovers_txt");

	# Strip perl-files:
	{
		local $/;
		undef $/;
		$perlfiles = `pkg_info -qL perl`;
	}
	($perlfiles_pattern = $perlfiles) =~ s/\n/|/g;
	$perlfiles_pattern =~ s/|$//;

	open (LEFT, $leftovers_txt) or die "can't read $leftovers_txt: $!";
	@left = <LEFT>;
	close (LEFT);
	@leftovers = grep(!/^(${perlfiles_pattern})$/, @left);

	open (LEFT, ">$leftovers_txt") or die "can't write $leftovers_txt: $!";
	print LEFT @leftovers;
	close (LEFT);

	if (scalar(@leftovers)) {
		# Store leftovers, for easier identification:
		system("$GTAR -T $leftovers_txt -plcf - | (cd leftovers-${arch}; $GTAR -plxf -)");
	}

	# Add links to leftover list:
	open (OUT, "> $leftovers_html")
		or die "can't write $leftovers_html";
	print OUT <<EOOUT;
<html>
<body>
<pre>
EOOUT
	foreach (@leftovers) {
		chomp;
	    	print OUT "<a href=\"${FTPHOST}/${FTPURL}/leftovers-${arch}$_\">$_</a>\n";
	}
	print OUT <<EOOUT2;
</pre>
</body>
</html>
EOOUT2
    	close(OUT);
}

# print the result of a single broken package
sub pkgResult {
	my ($pinfo, $state) = @_;
	my $pkg = $pinfo->{pkg};
	my $nbrokenby = $pinfo->{nbrokenby};
	my $nerrors = $pinfo->{nerrors};

	@idents = `${FIND} ${USR_PKGSRC}/${pkg} -type f -print | xargs grep \\\$NetBSD`;
	$datetime = "";
	$file = "";
	$ver = "";
	foreach $ident (@idents) {
		$ident =~ /\$[N]etBSD: ([^ ]*),v ([^ ]*) ([^ ]*) ([^ ]*) ([^ ]*) Exp \$/;
		if ("$3 $4" gt $datetime) {
			$datetime = "$3 $4";
			$file = $1;
			$ver = $1;
		}
	}

	$maintainer = `grep ^MAINTAINER $USR_PKGSRC/$pkg/Makefile`;
	$maintainer =~ s/MAINTAINER=[ \t]*//;
	$maintainer =~ s/</&lt;/g;
	$maintainer =~ s/>/&gt;/g;
	chomp($maintainer);

	(my $state_style = $state) =~ s/ //g;

	$nbrokenby_html = '<td>&nbsp;</td>'; 
	$nbrokenby_html =
		'<td align="right" class="pkg-'.$state_style.'">'.$nbrokenby.'</td>'
		if $nbrokenby > 0;

	if ($pinfo->{nerrors} != 0 && $verbose && $state eq "broken") {
		print swrite($report_form, $pkg, $nbrokenby > 0 ? $nbrokenby : "", $maintainer);
	}

	return <<EOHTML;
<tr>
  <td><a class="pkg-$state_style" href="$pinfo->{bf}" title="build log for $pkg">$pkg</a></td>
  $nbrokenby_html
  <td>$file</td>
  <td>$maintainer</td>
</tr>

EOHTML
}

# write the build report
sub writeReport {
	my $broken = getBroken();
	$nbroken = scalar(@{$broken->{"broken"}});
	$nbrokendep = scalar(@{$broken->{"broken depends"}});
	$nunpackaged = scalar(@{$broken->{"not packaged"}});
	$nbrokentot = $nbroken + $nbrokendep;
	$ntotal = $nunpackaged + $nbroken + $nbrokendep;

	# determine the number of packages attempted, and then successful
	open(ORDER, $ordfile) || die "can't open $ordfile: $!";
	my @order = <ORDER>;
	close(ORDER);
	my $nattempted = scalar(@order);
	my $nsuccessful = $nattempted - $ntotal;

	if ($verbose) {
		print <<EOF;
pkgsrc bulk build results
$os $osrev/$arch

Summary:

EOF
		my $summary_form = <<EOF;
        @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<  @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<
EOF
		print swrite($summary_form, "Build started:", $startdate);
		print swrite($summary_form, "Build ended:", $enddate);
		print swrite($summary_form);
		print swrite($summary_form, "Successfully packaged:", $nsuccessful);
		print swrite($summary_form, "Packages really broken:", $nbroken);
		print swrite($summary_form, "Pkgs broken due to them:", $nbrokendep);
		print swrite($summary_form, "Total broken:", $nbrokentot);
		print swrite($summary_form, "Not packaged:", $nunpackaged);
		print swrite($summary_form, "Total:", $ntotal);
		print <<EOF;

Packages not listed here resulted in a binary package. The build
report, including logs of failed/not-packaged is available from:

$FTPHOST/$FTPURL/$reportf
EOF
	}

	open(HTML,">$REPORT") or die "Can't write $REPORT: $!\n";
	print HTML <<EOHTML;
<html>
<head>
<title>$os $osrev/$arch bulk package build</title>
<style type="text/css">
<!--

body {
	Font-Family: Tahoma, Verdana, sans-serif;
	Line-Height: 1.3em;
	Text-Decoration: None;
	Color: black;
	Background-Color: white;
	Border-Width: 0;
}

table {
	Border-Width: 0;
}

table td {
	Font-Family: Tahoma, Verdana, sans-serif;
	line-height: 1em;
}

a:link {
	Color: #3535c5;
}

a:visited {
	Color: #700080;
}

a:hover {
	Color: #6565e5;
	Text-Decoration: underline;
}

tr {
	Vertical-Align: top;
}

td {
	Vertical-Align: top;
}

h1 {
	Font-Size: 3.5ex;
	Line-Height: 1em;
	Color: #000066;
}

h2 {
	Font-Size: 2.5ex;
	Line-Height: 1em;
	Color: #660000;
}

h3 {
	Font-Size: 2ex;
	Color: #660066;
}

h4 {
	Font-Size: 1.8ex;
	Color: #006600;
}

tt.filename {
	Line-Height: 1.3em;
	Color: #AA0000;
}

.pkgname {
	Font-Family: Arial, Helvetica, Courier, fixed;
	Font-Style: Italic;
	Text-Decoration: none;
	Line-Height: 1.3em;
}

.pkg-broken {
	Color: red;
}

.pkg-brokendepends {
	Color: orange;
}

.pkg-notpackaged {
	Color: blue;
}
-->
</style>
</head>

<body bgcolor="white" text="black" link="#3535c5" vlink="#700080" alink="#3535c5">

<a name="top"/>
<h1>pkgsrc bulk build results</h1>
<h2>$os $osrev/$arch</h2>

<h3>Summary</h3>

<table>
<tr>
  <td>Build started:			<td align="right">$startdate</td>
</tr>
<tr>
  <td>Build ended:			<td align="right">$enddate</td>
</tr>
<tr>
  <td>&nbsp;</td>			<td>&nbsp;</td>
</tr>
<tr>
  <td>Successfully packaged:</td>	<td align="right">$nsuccessful</td>
</tr>
<tr class="pkg-broken">
  <td>Packages really broken:</td>	<td align="right">$nbroken</td>
</tr>
<tr class="pkg-brokendepends">
  <td>Packages broken due to them:</td>	<td align="right">$nbrokendep</td>
</tr>
<tr>
  <td>Total broken:</td>		<td align="right">$nbrokentot</td>
</tr>
<tr class="pkg-notpackaged">
  <td>Not packaged:</td>		<td align="right">$nunpackaged</td>
</tr>
<tr>
  <td>Total:</td>			<td align="right">$ntotal</td>
</tr>
</table>

<p>
  Packages not listed here resulted in a <a
  href="../../packages/" title="binary packages for $os $osrev/$arch">binary
  package</a>. Results of failed packages are available below.
</p>

<p>
  Files leftover from the build (because of broken PLISTs, etc.) can be
  found in <a href="leftovers-${arch}.html" title="leftover files">this
  list</a>.
</p>

<p>
  Jump to:<br/>
  <ul>
    <li><a href="#broken">Broken packages</a></li>
    <li><a href="#broken depends">Broken dependencies</a></li>
    <li><a href="#not packaged">Not packaged</a></li>
  </ul>
</p>

EOHTML

	my %state_head = (
		"broken" => "Broken packages",
		"broken depends" => "Broken dependencies",
		"not packaged" => "Not packaged"
	);

	# only include broken packages in the email, see also pkgResult().
	if ($verbose && scalar(@{$broken->{"broken"}})) {
		print "\n\n$state_head{broken}\n\n";
		print $report_head;
	}

	foreach my $state ("broken", "broken depends", "not packaged") {
		next unless scalar(@{$broken->{$state}});

		print HTML <<EOHTML;

<a name="$state"/>
<h2>$state_head{$state}</h2>
<table width="100%">
<tr align="left">
  <th width="30%">Package</th>
  <th>Breaks</th>
  <th>File touched last</th>
  <th>Maintainer</th>
</tr>

EOHTML
		foreach my $pinfo (@{$broken->{$state}}) {
			print HTML pkgResult($pinfo, $state);
		}

		print HTML <<EOHTML;
</table>
<hr>
<a href="#top">Up to top</a><br/>
<hr>
EOHTML
	}

	print HTML <<EOHTML;
<hr>
<p>
The following cache files were used during the build:
</p>
<ul>
<li>The <a href="$bulkdbfile">SPECIFIC_PKGS bulk database file</a>.</li>
<li>The <a href="$dtfile">depends tree file</a>.</li>
<li>The <a href="$depfile">depends file</a>.</li>
<li>The <a href="$supfile">supports file</a>.</li>
<li>The <a href="$indfile">index file</a>.</li>
<li>The <a href="$ordfile">build order file</a>.</li>
</ul>
<hr>

<p>    
<ul>
<!-- <li>View the <a href="$prog">progress</a> of the build. -->
<!-- <li>See the list of <a href="../index.html">all log files</a>. -->
<li>Visit the <a href="http://www.NetBSD.org">NetBSD web site</a>.
<li>Learn more about
    <a href="http://www.NetBSD.org/Documentation/software/packages.html">
    The NetBSD Packages Collection</a>.
</ul>
</p>
</body>
</html>
EOHTML
	close(HTML);

	if ($verbose) {
		print "\n\n$ADMINSIG\n\n";
		print "[* This message was created by the Packages Collection bulk build software *]\n";
	}
}

# get and sort the broken packages
sub getBroken {
	if (!open (BF, $BROKENFILE)) {
		return undef;
	}
		
	my @in = <BF>;
	close (BF);

	my $res = {};
	foreach (@in) {
		chomp;
		my ($nerrors, $bf, $nbrokenby) = split;
		(my $pkg = $bf) =~ s,/$BROKENFILE,,;
		my %tmp = (
			bf => $bf,
			pkg => $pkg,
			nbrokenby => $nbrokenby,
			nerrors => $nerrors,
		);

		if ($nerrors > 0) {
			push(@{$res->{"broken"}}, \%tmp);
		} elsif ($nerrors == -1) {
			push(@{$res->{"broken depends"}}, \%tmp);
		} else {
			push(@{$res->{"not packaged"}}, \%tmp);
		}
	}

	# sort pkgs in each state
	foreach my $state ("broken", "broken depends", "not packaged") {
		$res->{$state} = [ sort { $a->{pkg} cmp $b->{pkg} } @{$res->{$state}} ];
	}

	return $res;
}

sub swrite {
	my $format = shift;
	$^A = "";
	formline($format, @_);
	$^A =~ s/\n\n/\n/;
	return $^A;
}