Skip to content

Commit

Permalink
[backend] implement obscpio diffing
Browse files Browse the repository at this point in the history
  • Loading branch information
@mlschroe
mlschroe committed Apr 4, 2016
1 parent c3a62e9 commit 7b8aabd
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 36 deletions.
170 changes: 137 additions & 33 deletions src/backend/BSSrcdiff.pm
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,90 @@ sub listgem {
return @gem;
}

sub cpiomode {
my ($m) = @_;
my $mm = '';
my $b = 0x100;
for (qw{r w x r w x r w x}) {
$mm .= $m & $b ? $_ : '-';
$b >>= 1;
}
substr($mm, 2, 1) = substr($mm, 2, 1) eq 'x' ? 'S' : 's' if $m & 0x800;
substr($mm, 5, 1) = substr($mm, 5, 1) eq 'x' ? 'S' : 's' if $m & 0x400;
substr($mm, 8, 1) = substr($mm, 8, 1) eq 'x' ? 'T' : 'T' if $m & 0x200;
return $mm;
}

sub listextractcpio {
my ($cpio, $cp) = @_;

my @c;
local *F;
local *G;
open(F, '<', $cpio) || die("$cpio: $!\n");
while (1) {
my $cpiohead;
die("cpio read error head\n") unless (read(F, $cpiohead, 110) || 0) == 110;
die("cpio: not a newc cpio\n") unless substr($cpiohead, 0, 6) eq '070701';
my $mode = hex(substr($cpiohead, 14, 8));
my $mtime = hex(substr($cpiohead, 46, 8));
my $fsize = hex(substr($cpiohead, 54, 8));
my $nsize = hex(substr($cpiohead, 94, 8));
die("ridiculous long filename\n") if $nsize > 8192;
my $nsizepad = 0;
$nsizepad = 4 - ($nsize + 2 & 3) if $nsize + 2 & 3;
my $name;
die("cpio read error name\n") unless (read(F, $name, $nsize + $nsizepad) || 0) == $nsize + $nsizepad;
$name = substr($name, 0, $nsize);
$name =~ s/\0.*//s;
my $type = $mode & 0xf000;
if ($type == 0x4000) {
$type = 'd';
} elsif ($type == 0x2000) {
$type = 'c';
} elsif ($type == 0x4000) {
$type = 'b';
} elsif ($type == 0x8000) {
$type = '-';
} elsif ($type == 0xa000) {
$type = 'l';
} else {
$type = '?';
}
last if !$fsize && $name eq 'TRAILER!!!';
push @c, {'type' => $type, 'name' => $name, 'size' => $fsize, 'mode' => cpiomode($mode)};
my $x = shift @$cp if $cp;
die if $cp && !$x;
undef $x if $x && !$x->{'extract'};
die("ridiculous long symlink\n") if $type eq 'l' && $fsize > 8192;
my $fsizepad = 0;
$fsizepad = 4 - ($fsize & 3) if $fsize & 3;
my $info = $type eq 'l' ? '' : 'd41d8cd98f00b204e9800998ecf8427e';
if ($x) {
open(G, '>', $x->{'extract'}) || die("$x->{'extract'}: $!\n");
}
if ($fsize > 0) {
my $ctx = Digest::MD5->new;
while ($fsize > 0) {
my $chunk = $fsize > 16384 ? 16384 : $fsize;
my $data;
die("cpio read error body\n") unless (read(F, $data, $chunk) || 0) == $chunk;
$ctx->add($data);
$info .= $data if $type eq 'l';
print G $data if $x;
$fsize -= $chunk;
}
$info = $ctx->hexdigest() unless $type eq 'l';
}
close(G) if $x;
$c[-1]->{'info'} = $info if $type eq '-' || $type eq 'l';
die("cpio read error bodypad\n") unless (read(F, $name, $fsizepad) || 0) == $fsizepad;
}
close(F);
return @c;
}


#
# diff file f1 against file f2
#
Expand Down Expand Up @@ -356,14 +440,27 @@ sub stripfirstdir {
$_->{'sname'} =~ s/^[^\/]*\/?// for @$l;
}

sub listit {
my ($f) = @_;
return listgem($f) if $f =~ /\.gem$/;
return listextractcpio($f) if $f =~ /\.obscpio$/;
return listtar($f);
}

sub extractit {
my ($f, $cp) = @_;
return listextractcpio($f, $cp || []) if $f =~ /\.obscpio$/;
return extracttar($f, $cp);
}

sub tardiff {
my ($f1, $f2, %opts) = @_;

my $max = $opts{'tmax'};
my $edir = $opts{'edir'};

my @l1 = ($f1 =~ /\.gem$/) ? listgem($f1) : listtar($f1);
my @l2 = ($f2 =~ /\.gem$/) ? listgem($f2) : listtar($f2);
my @l1 = listit($f1);
my @l2 = listit($f2);

die unless $edir;
for (@l1, @l2) {
Expand Down Expand Up @@ -434,8 +531,8 @@ sub tardiff {
if (! -d $edir) {
mkdir($edir) || die("mkdir $edir: $!\n");
}
extracttar($f1, \@l1) if $e1cnt;
extracttar($f2, \@l2) if $e2cnt;
extractit($f1, \@l1) if $e1cnt;
extractit($f2, \@l2) if $e2cnt;
}
my $lcnt = 0;
my $d = '';
Expand Down Expand Up @@ -483,7 +580,7 @@ my @simclasses = (
'dsc',
'changes',
'(?:diff?|patch)(?:\.gz|\.bz2|\.xz)?',
'(?:tar|tar\.gz|tar\.bz2|tar\.xz|tgz|tbz|gem)',
'(?:tar|tar\.gz|tar\.bz2|tar\.xz|tgz|tbz|gem|obscpio)',
);

sub findsim {
Expand Down Expand Up @@ -558,6 +655,13 @@ sub findsim {
return \%sim;
}


sub fn {
my ($dir, $f, $md5) = @_;
return $dir->($f, $md5) if ref($dir);
return "$dir/$md5-$f";
}

sub srcdiff {
my ($pold, $old, $orev, $pnew, $new, $rev, %opts) = @_;

Expand Down Expand Up @@ -605,11 +709,11 @@ sub srcdiff {
my $of = $f;
delete $xold{$of};
next if $old->{$of} eq $new->{$f};
my $r = filediff("$pold/$old->{$of}-$of", "$pnew/$new->{$f}-$f", %opts, 'diffarg' => $arg, 'fmax' => undef);
my $r = filediff(fn($pold, $of, $old->{$of}), fn($pnew, $f, $new->{$f}), %opts, 'diffarg' => $arg, 'fmax' => undef);
$d .= adddiffheader($r, $of, $f);
} else {
$d .= "\n++++++ new $extra file:\n";
my $r = filediff(undef, "$pnew/$new->{$f}-$f", %opts, 'diffarg' => $arg, 'fmax' => undef);
my $r = filediff(undef, fn($pnew, $f, $new->{$f}), %opts, 'diffarg' => $arg, 'fmax' => undef);
$d .= adddiffheader($r, undef, $f);
}
}
Expand Down Expand Up @@ -637,9 +741,9 @@ sub srcdiff {
next if $old->{$of} eq $new->{$f};
$d .= "\n++++++ $f\n" if $of eq $f;
}
if ($f =~ /\.(?:tgz|tar\.gz|tar\.bz2|tbz|tar\.xz|gem)$/) {
if ($f =~ /\.(?:tgz|tar\.gz|tar\.bz2|tbz|tar\.xz|gem|obscpio)$/) {
if (defined $of) {
my @r = tardiff("$pold/$old->{$of}-$of", "$pnew/$new->{$f}-$f", %opts);
my @r = tardiff(fn($pold, $of, $old->{$of}), fn($pnew, $f, $new->{$f}), %opts);
for my $r (@r) {
$d .= adddiffheader($r, $r->{'name'}, $r->{'name'});
}
Expand All @@ -650,11 +754,11 @@ sub srcdiff {
}
}
if (defined $of) {
my $r = filediff("$pold/$old->{$of}-$of", "$pnew/$new->{$f}-$f", %opts);
my $r = filediff(fn($pold, $of, $old->{$of}), fn($pnew, $f, $new->{$f}), %opts);
$d .= adddiffheader($r, $of, $f);
} else {
$d .= "\n++++++ $f (new)\n";
my $r = filediff(undef, "$pnew/$new->{$f}-$f", %opts);
my $r = filediff(undef, fn($pnew, $f, $new->{$f}), %opts);
$d .= adddiffheader($r, $of, $f);
}
}
Expand Down Expand Up @@ -683,22 +787,22 @@ sub udiff {
push @changed, $_;
}
}
my $orevb = defined($orev) ? " (revision $orev)" : '';
my $revb = defined($rev) ? " (revision $rev)" : '';
my $orevb = $orev && defined($orev->{'rev'}) ? " (revision $orev->{'rev'})" : '';
my $revb = $rev && defined($rev->{'rev'}) ? " (revision $rev->{'rev'})" : '';
my $d = '';
for my $f (@changed) {
$d .= "Index: $f\n" . ("=" x 67) . "\n";
my $r = filediff("$pold/$old->{$f}-$f", "$pnew/$new->{$f}-$f", %opts);
my $r = filediff(fn($pold, $f, $old->{$f}), fn($pnew, $f, $new->{$f}), %opts);
$d .= adddiffheader($r, "$f$orevb", "$f$revb");
}
for my $f (@added) {
$d .= "Index: $f\n" . ("=" x 67) . "\n";
my $r = filediff(undef, "$pnew/$new->{$f}-$f", %opts);
my $r = filediff(undef, fn($pnew, $f, $new->{$f}), %opts);
$d .= adddiffheader($r, "$f (added)", "$f$revb");
}
for my $f (@deleted) {
$d .= "Index: $f\n" . ("=" x 67) . "\n";
my $r = filediff("$pold/$old->{$f}-$f", undef, %opts);
my $r = filediff(fn($pold, $f, $old->{$f}), undef, %opts);
$d .= adddiffheader($r, "$f$orevb", "$f (deleted)");
}
return $d;
Expand All @@ -722,23 +826,23 @@ sub datadiff {
$of = $sim->{$f} if defined $sim->{$f};
$done{$of} = 1;
if (!defined($old->{$of})) {
my @s = stat("$pnew/$new->{$f}-$f");
my $r = filediff(undef, "$pnew/$new->{$f}-$f", %opts);
my @s = stat(fn($pnew, $f, $new->{$f}));
my $r = filediff(undef, fn($pnew, $f, $new->{$f}), %opts);
delete $r->{'state'};
push @added, {'state' => 'added', 'diff' => $r, 'new' => {'name' => $f, 'md5' => $new->{$f}, 'size' => $s[7]}};
} elsif ($old->{$of} ne $new->{$f}) {
if ($opts{'doarchive'} && $f =~ /\.(?:tgz|tar\.gz|tar\.bz2|tbz|tar\.xz|gem)$/) {
my @r = tardiff("$pold/$old->{$of}-$of", "$pnew/$new->{$f}-$f", %opts);
if ($opts{'doarchive'} && $f =~ /\.(?:tgz|tar\.gz|tar\.bz2|tbz|tar\.xz|gem|obscpio)$/) {
my @r = tardiff(fn($pold, $of, $old->{$of}), fn($pnew, $f, $new->{$f}), %opts);
if (@r == 0 && $f ne $of) {
# (almost) identical tars but renamed
my @os = stat("$pnew/$old->{$of}-$of");
my @s = stat("$pnew/$new->{$f}-$f");
my @os = stat(fn($pold, $of, $old->{$of}));
my @s = stat(fn($pnew, $f, $new->{$f}));
push @changed, {'state' => 'renamed', 'old' => {'name' => $of, 'md5' => $old->{$of}, 'size' => $os[7]}, 'new' => {'name' => $f, 'md5' => $new->{$f}, 'size' => $s[7]}};
}
if (@r == 1 && !$r[0]->{'old'} && !$r[0]->{'new'}) {
# tardiff was too big
my @os = stat("$pnew/$old->{$of}-$of");
my @s = stat("$pnew/$new->{$f}-$f");
my @os = stat(fn($pold, $of, $old->{$of}));
my @s = stat(fn($pnew, $f, $new->{$f}));
push @changed, {'state' => 'changed', 'diff' => $r[0], 'old' => {'name' => $of, 'md5' => $old->{$of}, 'size' => $os[7]}, 'new' => {'name' => $f, 'md5' => $new->{$f}, 'size' => $s[7]}};
@r = ();
}
Expand All @@ -754,21 +858,21 @@ sub datadiff {
delete $r->{'new'};
}
} else {
my @os = stat("$pnew/$old->{$of}-$of");
my @s = stat("$pnew/$new->{$f}-$f");
my $r = filediff("$pold/$old->{$of}-$of", "$pnew/$new->{$f}-$f", %opts);
my @os = stat(fn($pold, $of, $old->{$of}));
my @s = stat(fn($pnew, $f, $new->{$f}));
my $r = filediff(fn($pold, $of, $old->{$of}), fn($pnew, $f, $new->{$f}), %opts);
delete $r->{'state'};
push @changed, {'state' => 'changed', 'diff' => $r, 'old' => {'name' => $of, 'md5' => $old->{$of}, 'size' => $os[7]}, 'new' => {'name' => $f, 'md5' => $new->{$f}, 'size' => $s[7]}};
}
} elsif ($f ne $of) {
my @os = stat("$pnew/$old->{$of}-$of");
my @s = stat("$pnew/$new->{$f}-$f");
my @os = stat(fn($pold, $of, $old->{$of}));
my @s = stat(fn($pnew, $f, $new->{$f}));
push @changed, {'state' => 'renamed', 'old' => {'name' => $of, 'md5' => $old->{$of}, 'size' => $os[7]}, 'new' => {'name' => $f, 'md5' => $new->{$f}, 'size' => $s[7]}};
}
}
for my $of (grep {!$done{$_}} sort(keys %$old)) {
my @os = stat("$pnew/$old->{$of}-$of");
my $r = filediff("$pold/$old->{$of}-$of", undef, %opts);
my @os = stat(fn($pold, $of, $old->{$of}));
my $r = filediff(fn($pold, $of, $old->{$of}), undef, %opts);
delete $r->{'state'};
push @added, {'state' => 'deleted', 'diff' => $r, 'old' => {'name' => $of, 'md5' => $old->{$of}, 'size' => $os[7]}};
}
Expand Down Expand Up @@ -815,12 +919,12 @@ sub issuediff {
my %oldchanges;
my %newchanges;
for my $f (grep {/\.changes$/} sort(keys %$old)) {
for (split(/------------------------------------------+/, readstr("$pold/$old->{$f}-$f"))) {
for (split(/------------------------------------------+/, readstr(fn($pold, $f, $old->{$f})))) {
$oldchanges{Digest::MD5::md5_hex($_)} = $_;
}
}
for my $f (grep {/\.changes$/} sort(keys %$new)) {
for (split(/------------------------------------------+/, readstr("$pnew/$new->{$f}-$f"))) {
for (split(/------------------------------------------+/, readstr(fn($pnew, $f, $new->{$f})))) {
$newchanges{Digest::MD5::md5_hex($_)} = $_;
}
}
Expand Down
18 changes: 15 additions & 3 deletions src/backend/bs_srcserver
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5402,21 +5402,33 @@ sub sourcediff {
}
my $tmpdir = "$uploaddir/srcdiff$$";
my $d;
my %xobscpio;
mkdir_p($uploaddir);
my $xobscpio = sub {
return repfilename($_[0], $_[1], $_[2]) if $_[1] !~ /\.obscpio$/;
my $tmp = "$uploaddir/sourcediff.obscpio.$$.$_[2]-$_[1]";
copyonefile_tmp($_[0]->{'project'}, $_[0]->{'package'}, $_[1], $_[2], $tmp) unless $xobscpio{$tmp};
$xobscpio{$tmp} = 1;
return $tmp;
};
my $ofn = sub { $xobscpio->($orev, $_[0], $_[1]) };
my $fn = sub { $xobscpio->($rev, $_[0], $_[1]) };
if ($view eq 'xml') {
my %opts = ('edir' => $tmpdir, 'similar' => 1, 'doarchive' => 1, 'fmax' => $fmax, 'tmax' => $tmax);
if (!$cgi->{'onlyissues'}) {
$xmlret->{'files'} = { 'file' => BSSrcdiff::datadiff("$srcrep/$opackid", $ofiles, $orev->{'rev'}, "$srcrep/$packid", $files, $rev->{'rev'}, %opts) };
$xmlret->{'files'} = { 'file' => BSSrcdiff::datadiff($ofn, $ofiles, $orev, $fn, $files, $rev, %opts) };
}
if ($cgi->{'withissues'}) {
my $trackers = readxml("$BSConfig::bsdir/issuetrackers.xml", $BSXML::issue_trackers, 1) || {};
$trackers = $trackers->{'issue-tracker'} || [];
$xmlret->{'issues'} = { 'issue' => BSSrcdiff::issuediff("$srcrep/$opackid", $ofiles, $orev->{'rev'}, "$srcrep/$packid", $files, $rev->{'rev'}, $trackers, %opts) };
$xmlret->{'issues'} = { 'issue' => BSSrcdiff::issuediff($ofn, $ofiles, $orev, $fn, $files, $rev, $trackers, %opts) };
}
BSUtil::data2utf8xml($xmlret);
$d = XMLout($BSXML::sourcediff, $xmlret);
} else {
$d = BSSrcdiff::diff("$srcrep/$opackid", $ofiles, $orev->{'rev'}, "$srcrep/$packid", $files, $rev->{'rev'}, $fmax, $tmax, $tmpdir, $view eq 'unified' ? 1 : 0);
$d = BSSrcdiff::diff($ofn, $ofiles, $orev, $fn, $files, $rev, $fmax, $tmax, $tmpdir, $view eq 'unified' ? 1 : 0);
}
unlink($_) for keys %xobscpio;
mkdir_p("$diffcache/".substr($cacheid, 0, 2));
writestr("$diffcache/.new$$", $cn, $d);
unlink("$cn.run");
Expand Down

0 comments on commit 7b8aabd

Please sign in to comment.