diff --git a/MANIFEST b/MANIFEST index e8b0dcb..0160c2f 100644 --- a/MANIFEST +++ b/MANIFEST @@ -8,4 +8,3 @@ BUGS TODO Changes Makefile.PL -lib/Intrusive.pm diff --git a/bin/batch.pl b/bin/batch.pl deleted file mode 100644 index a9c8bb9..0000000 --- a/bin/batch.pl +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/perl -use strict; -use warnings; -use 5.010; -use Data::Dumper; -use Getopt::Long; - -GetOptions( - "debug" => \my $debug, - "dry" => \my $dry, - "help|h" => \my $help, -) # flag -or die "Error in command line arguments"; - -if ($help) { - print <<'EOM'; -Usage: - batch.pl /path/to/packages 0 50 # process the first 51 packages -EOM - exit; -} - -my ($dir, $from, $to) = @ARGV; -$from ||= 0; -$to ||= $from; - -my @skip = qw/ - perl-AcePerl - perl-Acme-MetaSyntactic - perl-Acme-Ook - perl-Algorithm-Munkres - perl-Alien-LibGumbo - perl-Alien-SVN - perl-Alien-Tidyp - perl-Apache-AuthNetLDAP - perl-Apache-Filter - perl-Apache-SessionX - perl-Apache-Gallery - perl-App-ProcIops - perl-App-Nopaste - perl-App-SVN-Bisect - perl-App-gcal - perl-Array-Dissect - perl-Audio-CD - perl-Authen-SASL-Cyrus - perl-BIND-Conf_Parser - perl-BSXML - perl-Boost-Geometry-Utils - perl-Class-Accessor-Chained - Class-Multimethods - perl-Crypt-HSXKPasswd - perl-Crypt-Rot13 -/; -my %skip; -@skip{ @skip } = (); - -opendir my $dh, $dir or die $!; -my @pkgs = sort grep { - -d "$dir/$_" && m/^perl-/ - and not exists $skip{ $_ } -} readdir $dh; -closedir $dh; - -my $count = @pkgs; -say "Total: $count"; - -my $opt_debug = $debug ? '--debug' : ''; -for my $i ($from .. $to) { - my $pkg = $pkgs[ $i ]; - chdir $dir; - say "=========== ($i) $pkg"; - next if $dry; - chdir $pkg; - my $mod = $pkg; - $mod =~ s/^perl-//; - my @glob = glob("$mod*"); - my $cmd = qq{cpanspec $opt_debug -v -f --skip-changes --pkgdetails /tmp/02packages.details.txt.gz @glob 2>&1}; - say "Cmd: $cmd"; - my $out = qx{$cmd}; - say $out; - -} diff --git a/bin/build-status b/bin/build-status deleted file mode 100755 index 91dcd9d..0000000 --- a/bin/build-status +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env perl -# Shows the OBS build status statistics for a certain project. -use strict; -use warnings; -use 5.010; - -use FindBin '$Bin'; -use lib "$Bin/../lib"; -use CPAN2OBS qw/ debug info prompt /; -use XML::Simple qw/ XMLin /; -use YAML::XS; - -use Getopt::Long; -GetOptions( - "project=s" => \my $project, - "project-prefix=s" => \my $project_prefix, - "yaml" => \my $as_yaml, - "lastbuild" => \my $lastbuild, - "repo=s" => \my $repo, - "arch=s" => \my $arch, - "help|h" => \my $help, -); -usage(), exit if $help; - -$arch ||= 'x86_64'; -$repo ||= 'openSUSE_Tumbleweed'; -my $apiurl = "https://api.opensuse.org"; - -my $cpan2obs = CPAN2OBS->new({ - apiurl => $apiurl, -}); - - -my %counts; - -if ($project_prefix) { - my @letters = ('A' .. 'Z'); - - if (@ARGV) { - @letters = map { uc } @ARGV; - } - - for my $letter (@letters) { - my $project = "$project_prefix$letter"; - build_status($cpan2obs, $project, $letter, \%counts); - } -} -elsif ($project) { - my ($key) = @ARGV; - $key //= "key"; - build_status($cpan2obs, $project, $key, \%counts); -} - -if ($as_yaml) { - say YAML::XS::Dump \%counts; - exit; -} - -my @states = qw/ - total building finished scheduled blocked broken - succeeded failed unresolvable disabled excluded -/; -my @states_title = qw/ - total building finished scheduled blocked broken - succeeded failed unres. disabled excluded -/; - -my $fmt = join ' | ', ('%9s') x @states; -info sprintf "%10s | $fmt", '', @states_title; -for my $letter (sort keys %counts) { - my $localcounts = $counts{ $letter }; - info sprintf "%10s | $fmt", $letter, map { $_ || 0 } @$localcounts{ @states }; -} - -sub build_status { - my ($self, $project, $letter, $counts) = @_; - my $apiurl = $self->apiurl; - my $args = "repository=$repo&arch=$arch"; - if ($lastbuild) { - $args .= '&lastbuild=1'; - } - my $cmd = sprintf "osc -A %s api '/build/%s/_result?$args'", - $apiurl, $project; - debug("CMD $cmd"); - open my $fh, "-|", $cmd; - my $res = XMLin($fh, forcearray => [qw/status/]); - close $fh; - - my %localcounts; - if ($res && $res->{result}) { - my $result = $res->{result}; - if (ref $result ne 'ARRAY') { - $result = [$result]; - } - for my $r (@$result) { - my $packages = $r->{status}; - for my $pkg (@$packages) { - my $code = $pkg->{code}; - $localcounts{ $code }++; - $localcounts{total}++; - } - } - } - $counts->{ $letter } = \%localcounts; -} - -sub usage { - info <<"EOM"; -Usage: - - $0 --project devel:languages:perl:autoupdate --yaml autoupdate --repo standard - $0 --project devel:languages:perl --yaml perl - $0 --project devel:languages:perl perl --repo SLE_15 - $0 --project devel:languages:perl --yaml perl --repo openSUSE_Tumbleweed --arch ppc -EOM -} diff --git a/bin/fetch-cpan b/bin/fetch-cpan deleted file mode 100755 index bcb2c0b..0000000 --- a/bin/fetch-cpan +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env perl -# Will fetch module list from CPAN and update status files under -# $data/cpan/$letter.tsv -use strict; -use warnings; -use 5.010; - -use FindBin '$Bin'; -use lib "$Bin/../lib"; - -use Getopt::Long; -use CPAN2OBS qw/ debug info prompt /; - -GetOptions( - "data=s" => \my $data, - "help|h" => \my $help, -); -usage(), exit if $help; -die "--data missing" unless $data; -my $details_url = "http://www.cpan.org/modules/02packages.details.txt.gz"; - -my @skip = qw/ - Acme-DependOnEverything - Acme-Shining -/; -my %skip; -@skip{ @skip } = (1) x @skip; - -my $cpan2obs = CPAN2OBS->new({ - data => $data, - skip => \%skip, - cpandetails => $details_url, -}); - -$cpan2obs->fetch_cpan_list(); - -sub usage { - print <<"EOM"; -Usage: - - $0 --data - - $0 --data ~/obs-mirror -EOM -} diff --git a/bin/fetch-cpan.sh b/bin/fetch-cpan.sh deleted file mode 100755 index 3e3d3ec..0000000 --- a/bin/fetch-cpan.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -case "$1" in - -h|--help) - echo "Wrapper for bin/fetch-cpan" - exit - ;; -esac - -DIR="$( dirname ${BASH_SOURCE[0]} )/.." -cd $DIR - -./bin/fetch-cpan --data ~/obs-mirror - diff --git a/bin/status b/bin/status deleted file mode 100755 index b983dd3..0000000 --- a/bin/status +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; -use 5.010; - -use FindBin '$Bin'; -use lib "$Bin/../lib"; -use CPAN2OBS qw/ debug info prompt /; - -use Getopt::Long; -GetOptions( - "update" => \my $update, - "data=s" => \my $data, - "project=s" => \my $project, - "help|h" => \my $help, -); -usage(), exit if $help; -die "--data missing" unless $data; -$project ||= "home:tinita:cpan-mirror:CPAN-"; -#$project ||= "devel:languages:perl:CPAN-"; - -my $apiurl = "https://api.opensuse.org"; - -my @letters = ('A' .. 'Z'); - -if (@ARGV) { - @letters = map { uc } @ARGV; -} - -my $cpan2obs = CPAN2OBS->new({ - data => $data, - apiurl => $apiurl, - project_prefix => $project, -}); - -my @states = qw/ total done todo new error disabled /; -my @status_list; -for my $letter (@letters) { - if ($update) { - $cpan2obs->update_status($letter); - } - my $states = $cpan2obs->fetch_status($letter); - my %counts; - for my $dist (sort keys %$states) { - my $dist_status = $states->{ $dist }; - my $status = $dist_status->[0]; - my $obs_ok = $dist_status->[4]; - if ($status eq 'done' and not $obs_ok) { - $status = 'disabled'; - } - if ($status =~ m/^error/) { - $status = 'error'; - } - $counts{ $status }++; - $counts{total}++; - next; - } - - push @status_list, [ - $letter, map { $counts{ $_ } || 0 } @states, - ]; -} - -info sprintf " | %10s | %10s | %10s | %10s | %15s | %10s", @states; -for my $item (@status_list) { - info sprintf "%s | %10s | %10s | %10s | %10s | %15s | %10s", @$item; -} - -exit; - -sub usage { - print <<"EOM"; -Usage: - - $0 --data ~/obs-mirror --project devel:languages:perl:CPAN- A B - $0 --data ~/obs-mirror --project devel:languages:perl:CPAN- - $0 --data ~/obs-mirror --project devel:languages:perl:CPAN- --update - -Report status of modules to update. -With --update it will look into the latest data fetched with fetch-cpan` and -update the status. -EOM -} diff --git a/bin/status-perl b/bin/status-perl deleted file mode 100755 index c8036c3..0000000 --- a/bin/status-perl +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; -use 5.010; - -use FindBin '$Bin'; -use lib "$Bin/../lib"; -use CPAN2OBS qw/ debug info prompt /; - -use Getopt::Long; -GetOptions( - "update" => \my $update, - "data=s" => \my $data, - "project=s" => \my $project, - "help|h" => \my $help, -); -usage(), exit if $help; -die "--data missing" unless $data; -$project ||= "home:tinita:CPAN:autoupdate"; -#$project ||= "devel:languages:perl:autoupdate"; - -my $apiurl = "https://api.opensuse.org"; - -my $cpan2obs = CPAN2OBS->new({ - data => $data, - apiurl => $apiurl, - project_prefix => $project, -}); - -my @states = qw/ total done todo new older error /; -my @status_list; - -if ($update) { - $cpan2obs->update_status_perl(); -} -my $states = $cpan2obs->fetch_status_perl(); -my %counts; -for my $dist (sort keys %$states) { - my $dist_status = $states->{ $dist }; - my $status = $dist_status->[0]; - if ($status =~ m/^error/) { - $status = 'error'; - } - $counts{ $status }++; - $counts{total}++; - next; -} - -push @status_list, [ - "perl", map { $counts{ $_ } || 0 } @states, -]; - -info sprintf " | %10s | %10s | %10s | %10s | %10s | %10s", @states; -for my $item (@status_list) { - info sprintf "%s | %10s | %10s | %10s | %10s | %10s | %10s", @$item; -} - -exit; - -sub usage { - print <<"EOM"; -Usage: - - $0 --data ~/obs-mirror --project devel:languages:perl:autoupdate - $0 --data ~/obs-mirror --project devel:languages:perl:autoupdate --update - -Report status of modules to update. -With --update it will look into the latest data fetched with fetch-cpan` and -update the status. -EOM -} - diff --git a/bin/update b/bin/update deleted file mode 100755 index 0faa58b..0000000 --- a/bin/update +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; -use 5.010; - -use FindBin '$Bin'; -use lib "$Bin/../lib"; -use CPAN2OBS qw/ debug info prompt /; -use Data::Dumper; -use Getopt::Long; -my $max = 50; -GetOptions( - 'ask' => \my $ask, - 'ask-commit' => \my $ask_commit, - 'max=i' => \$max, - 'data=s' => \my $data, - 'project=s' => \my $project, - 'package=s@' => \my @packages, - 'redo' => \my $redo, - 'help|h' => \my $help, -); -usage(), exit if $help; -usage(), exit 1 if (not $project or not $data); -#$project ||= "home:tinita:cpan-mirror:CPAN-"; -#$project ||= "devel:languages:perl:CPAN-"; - -unless (@ARGV) { - usage(); - exit 1; -} -my ($letter) = @ARGV; -unless ($letter =~ m/^[A-Za-z]\z/) { - usage(); - exit 1; -} -$letter = uc $letter; - -my $cpan2obs = CPAN2OBS->new({ - data => $data, - cpanmirror => 'http://cpan.noris.de', - apiurl => 'https://api.opensuse.org', - cpanspec => "$Bin/../cpanspec", - project_prefix => $project, -}); - -info("Updating status $letter"); -$cpan2obs->update_status($letter); - -$cpan2obs->update_obs($letter, { - ask => $ask, - ask_commit => $ask_commit, - max => $max, - packages => \@packages, - redo => $redo, -}); - -info("Updating status $letter"); -$cpan2obs->update_status($letter); -exit; - -sub usage { - print <<"EOM"; -Usage: - $0 update [options] first-letter - - $0 --data ~/obs-mirror --project devel:languages:perl:CPAN- --max 20 A - # Only update one package - $0 --data ... --project ... --ask --ask-commit --package YAML-LibYAML - # Force if status is already done - $0 --data ... --project ... --ask --ask-commit --package YAML-LibYAML --redo - ---max Only to \$max updates, even if there are more packages to update ---ask Ask before trying to update ---ask-commit Ask before committing - -EOM -} - diff --git a/bin/update-autoupdate.sh b/bin/update-autoupdate.sh deleted file mode 100755 index 30469e1..0000000 --- a/bin/update-autoupdate.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -case "$1" in - -h|--help) - echo "Wrapper for bin/status-perl && bin/update-perl" - exit - ;; -esac - -DIR="$( dirname ${BASH_SOURCE[0]} )/.." -cd $DIR - -./bin/status-perl --data ~/obs-mirror --project devel:languages:perl:autoupdate --update - -./bin/update-perl \ - --data ~/obs-mirror \ - --project devel:languages:perl:autoupdate \ - --max 20 diff --git a/bin/update-dlp-cpan.sh b/bin/update-dlp-cpan.sh deleted file mode 100755 index 8741379..0000000 --- a/bin/update-dlp-cpan.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash - -case "$1" in - -h|--help) - echo "Wrapper for bin/status && bin/update" - exit - ;; -esac - -DIR="$( dirname ${BASH_SOURCE[0]} )/.." -cd $DIR - -./bin/status --data ~/obs-mirror --project devel:languages:perl:CPAN- --update - -for LETTER in A B C D E F G H I J K L M N O P Q R S T U V W X Y Z -do - echo $LETTER - ./bin/update \ - --data ~/obs-mirror \ - --project devel:languages:perl:CPAN- \ - --max 20 \ - $LETTER -done diff --git a/bin/update-perl b/bin/update-perl deleted file mode 100755 index 13800c5..0000000 --- a/bin/update-perl +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env perl -use strict; -use warnings; -use 5.010; - -use FindBin '$Bin'; -use lib "$Bin/../lib"; -use CPAN2OBS qw/ debug info prompt /; -use Data::Dumper; -use Getopt::Long; -my $max = 50; -GetOptions( - 'ask' => \my $ask, - 'ask-commit' => \my $ask_commit, - 'max=i' => \$max, - 'data=s' => \my $data, - 'project=s' => \my $project, - 'package=s@' => \my @packages, - 'redo' => \my $redo, - 'help|h' => \my $help, -); -usage(), exit if $help; -usage(), exit 1 if (not $project or not $data); -#$project ||= "home:tinita:CPAN:autoupdate"; -#$project ||= "devel:languages:perl:CPAN-"; - -my $cpan2obs = CPAN2OBS->new({ - data => $data, - cpanmirror => 'http://cpan.metacpan.org', - apiurl => 'https://api.opensuse.org', - cpanspec => "$Bin/../cpanspec", - project_prefix => $project, -}); - -my $letter = 'perl'; -info("Updating status $letter"); -$cpan2obs->update_status_perl; - -$cpan2obs->update_obs_perl({ - ask => $ask, - ask_commit => $ask_commit, - max => $max, - packages => \@packages, - redo => $redo, -}); - -info("Updating status $letter"); -$cpan2obs->update_status_perl; -exit; - -sub usage { - print <<"EOM"; -Usage: - $0 update [options] - - $0 --data ~/obs-mirror --project devel:languages:perl:autoupdate --max 20 - # Only update one package - $0 --data ... --project ... --ask --ask-commit --package YAML-LibYAML - # Force if status is already done - $0 --data ... --project ... --ask --ask-commit --package YAML-LibYAML --redo - ---max Only to \$max updates, even if there are more packages to update ---ask Ask before trying to update ---ask-commit Ask before committing - - -EOM -} - diff --git a/checkallcpan b/checkallcpan deleted file mode 100755 index 216cba2..0000000 --- a/checkallcpan +++ /dev/null @@ -1,12 +0,0 @@ -#! /bin/bash - -n=`mktemp` -s=`mktemp` -for l in {A..Z}; do - curl -s "https://api.opensuse.org/public/build/devel:languages:perl:CPAN-$l/_result?repository=openSUSE_Leap_42.2&arch=x86_64" | sed -e "s,nothing,\nnothing,g" >> $n - curl -s "https://api.opensuse.org/public/build/devel:languages:perl:CPAN-$l/_result?repository=openSUSE_Leap_42.2&arch=x86_64&code=succeeded" | grep package= | sed -e 's,.*package=",,; s,".*,,' >> $s -done -n2=`mktemp` -grep nothing $n | sed -e 's,.*nothing provides perl(,,; s,).*,,' | sort | uniq -c | sort -n > $n2 -for i in `sed -e 's,^perl-,,; s,-,::,g' $s` ; do grep " $i\$" $n2 ; done | sort -nr | head - diff --git a/lib/CPAN2OBS.pm b/lib/CPAN2OBS.pm deleted file mode 100644 index 187aa1b..0000000 --- a/lib/CPAN2OBS.pm +++ /dev/null @@ -1,853 +0,0 @@ -package CPAN2OBS; -use strict; -use warnings; -use 5.010; - -use base 'Exporter'; -our @EXPORT_OK = qw/ debug info prompt /; - -use Data::Dumper; -use Term::ANSIColor qw/ colored /; -use YAML::XS qw/ DumpFile /; -use XML::Simple qw/ XMLin /; -use Storable qw/ retrieve store /; -use File::Basename qw/ basename /; -use File::Copy qw/ copy move /; -use File::Path qw/ remove_tree /; -use File::Spec; -use Parse::CPAN::Packages; -use File::Glob qw/ bsd_glob /; -use FindBin '$Bin'; -use autodie qw/ mkdir chdir open close unlink /; - -use Moo; -use namespace::clean; - -has data => ( is => 'rw', coerce => \&_coerce_data ); -has cpandetails => ( is => 'ro' ); -has skip => ( is => 'ro' ); -has apiurl => ( is => 'ro' ); -has cpanmirror => ( is => 'ro' ); -has cpanspec => ( is => 'ro' ); -has project_prefix => ( is => 'ro', coerce => \&_coerce_project_prefix ); -has locked => ( is => 'rw' ); - -sub _coerce_data { - my ($data) = @_; - unless (File::Spec->file_name_is_absolute($data)) { - $data = File::Spec->rel2abs($data); - } - return $data; -} - -sub _coerce_project_prefix { - my ($prefix) = @_; - unless ($prefix =~ m/\A[A-Za-z][A-Za-z:-]+\z/) { - die "Invalid project prefix '$prefix'"; - } - return $prefix; -} - -sub debug { - my ($msg) = @_; - say STDERR colored([qw/ grey15 /], $msg); -} -sub info { - my ($msg) = @_; - say colored([qw/ magenta /], $msg); -} -sub prompt { - my ($msg) = @_; - print colored([qw/ cyan /], $msg); - chomp(my $answer = ); - $answer = uc $answer; - return $answer; -} - -sub fetch_status { - my ($self, $letter) = @_; - my $data = $self->data; - my $status_file = "$data/status/$letter.tsv"; - my %states; - unless (-e $status_file) { - return \%states; - } - $self->lockdata; - open my $fh, '<', $status_file; - while (my $line = <$fh>) { - chomp $line; - my ($dist, $status, $version, $url, $obs_tar, $obs_ok) = split /\t/, $line; - $states{ $dist } = [ $status, $version, $url, $obs_tar, $obs_ok ]; - } - close $fh; - $self->unlockdata; - return \%states; -} - -sub fetch_status_perl { - my ($self, $letter) = @_; - my $data = $self->data; - my $status_file = "$data/status/perl.tsv"; - my %states; - unless (-e $status_file) { - return \%states; - } - $self->lockdata; - open my $fh, '<', $status_file; - while (my $line = <$fh>) { - chomp $line; - my ($dist, $status, $version, $url, $obs_version) = split /\t/, $line; - $states{ $dist } = [ $status, $version, $url, $obs_version ]; - } - close $fh; - $self->unlockdata; - return \%states; -} - -sub write_status { - my ($self, $letter, $states) = @_; - my $data = $self->data; - mkdir "$data/status" unless -d "$data/status"; - my $status_file = "$data/status/$letter.tsv"; - open my $fh, '>', $status_file; - for my $dist (sort keys %$states) { - my $status = $states->{ $dist }; - say $fh join "\t", $dist, @$status; - } - close $fh; -} - -sub fetch_cpan_list { - my ($self) = @_; - $self->lockdata; - my $data = $self->data; - my $cpan_modules_dir = "$data/cpan"; - my $cpan_stats = "$cpan_modules_dir/stats.yaml"; - my $details = "$data/02packages.details.txt.gz"; - my $details_url = $self->cpandetails; - - my $details_mtime = -e $details ? (stat $details)[9] : 0; - my $stats_mtime = -e $cpan_stats ? (stat $cpan_stats)[9] : 0; - if ($details_mtime + 60 * 30 < time) { - my $cmd = "wget -q $details_url -O $details"; - debug("CMD $cmd"); - system $cmd; - $details_mtime = -e $details ? (stat $details)[9] : 0; - } - else { - info("$details uptodate"); - } - if ($stats_mtime >= $details_mtime) { - info("$cpan_stats uptodate"); - $self->unlockdata; - return 1; - } - info("Parsing $details"); - my $p = Parse::CPAN::Packages->new($details); - my %seen; - my %upstream; - for my $m ($p->packages) { - $m = $m->distribution; - my $url = $m->prefix; - my $dist = $m->dist; - unless ($dist) { - next; - } - if ($self->skip->{ $dist }) { - info("Ignoring $dist"); - next; - } - my $version = $m->version; - if (not $version) { - #warn sprintf "Distribution %s has no version defined", $url; - next; - } - my $first = uc substr $dist, 0, 1; - if ($dist eq 'XML-Xerces') { - # versions like '2.7.0-1' are not parseable. - # XML-Xerces is in the module index multiple times with different - # versions and that leads to a different version almost every day - $version =~ s/-//g; - } - my $v = eval { version->parse($version); }; - $v ||= version->declare('0'); - if (not exists $seen{ $dist } or ($seen{ $dist } ) < $v) { - $seen{ $dist } = $v; - $upstream{ $first }->{ $dist } = [$v, $url]; - } - } - - my %stats; - my $total = 0; - for my $letter ('A' .. 'Z') { - my $count = keys %{ $upstream{ $letter } }; - $total += $count; - $stats{ $letter } = $count; - $self->hash_to_cpan_file($letter, $upstream{ $letter }); - } - $stats{total} = $total; - DumpFile($cpan_stats, \%stats); - - system "cat $cpan_stats"; - $self->unlockdata; -} - -sub hash_to_cpan_file { - my ($self, $letter, $upstream) = @_; - my $data = $self->data; - my $cpan_modules_dir = "$data/cpan"; - mkdir $cpan_modules_dir unless -d $cpan_modules_dir; - my $cpan_modules_file = "$cpan_modules_dir/$letter.tsv"; - - open my $fh, '>', $cpan_modules_file; - for my $dist (sort keys %$upstream) { - my $info = $upstream->{ $dist }; - my ($version, $url) = @$info; - say $fh join "\t", ($dist, $version, $url); - } - close $fh; -} - -sub from_cpan_file { - my ($self, $letter) = @_; - my $data = $self->data; - my $cpan_modules_dir = "$data/cpan"; - mkdir $cpan_modules_dir unless -d $cpan_modules_dir; - my $cpan_modules_file = "$cpan_modules_dir/$letter.tsv"; - - open my $fh, '<', $cpan_modules_file; - my %upstream; - while (my $line = <$fh>) { - chomp $line; - my ($dist, $version, $url) = split m/\t/, $line; - $upstream{ $dist} = [ $version, $url ]; - } - close $fh; - return \%upstream; -} - -sub fetch_obs_info { - my ($self, $letter) = @_; - my $data = $self->data; - my $apiurl = $self->apiurl; - my $project_prefix = $self->project_prefix . $letter; - - my $cache = $self->fetch_obs_cache($letter); - - my $obsdir = "$data/obs"; - my $letter_xml = "$obsdir/CPAN-$letter.xml"; - mkdir $obsdir unless -d $obsdir; - my %obs_info; - - my $old =(not -e $letter_xml or ((stat $letter_xml)[9] + 60 * 60) < time); - if (1) { - my $cmd = "osc -A $apiurl api '/source/$project_prefix?view=info' >$letter_xml"; - debug("CMD $cmd"); - system $cmd; - } - if (not -s $letter_xml) { - return \%obs_info; - } - my $info = XMLin($letter_xml)->{sourceinfo}; - return \%obs_info unless $info; - $info = [$info] unless ref $info eq 'ARRAY'; - - mkdir "$data/project-xml" unless -d "$data/project-xml"; - for my $pi (@$info) { - my $srcmd5 = $pi->{srcmd5}; - my $package = $pi->{package}; - my $cached = $cache->{ $srcmd5 }; - if (not defined $cached) { - info("fetch obs xml for $package"); - my $cmd = sprintf "osc -A %s api /source/%s/%s >$data/project-xml/$package.xml", - $apiurl, $project_prefix, $package; - debug("CMD $cmd"); - system $cmd; - - unless (-s "$data/project-xml/$package.xml") { - warn __PACKAGE__.':'.__LINE__.": !!!! no osc data for $package\n"; - next; - } - my $pxml = XMLin "$data/project-xml/$package.xml"; - my $name = $pxml->{name}; - $obs_info{ $name }->{ok} = 1; - if ($pxml->{entry}->{'cpanspec.error'}) { - $obs_info{ $name }->{ok} = 0; - } - for my $entry (keys %{ $pxml->{entry} }) { - if ($entry =~ m/\.tar/ || $entry =~ m/\.tgz$/ || $entry =~ m/\.zip$/) { - $obs_info{ $name }->{archive} = $entry; - } - } - $cache->{ $srcmd5 } = $obs_info{ $package } || {}; - $self->store_obs_cache($letter, $cache); - } - else { - if (ref $cached) { - $obs_info{ $package } = $cached; - } - else { - $obs_info{ $package } = { archive => $cached, ok => 1 }; - } - } - } - return \%obs_info; -} - -sub fetch_obs_cache { - my ($self, $letter) = @_; - my $data = $self->data; - mkdir "$data/obs-cache" unless -d "$data/obs-cache"; - my $cachefile = "$data/obs-cache/$letter"; - my $cache = {}; - if (-f $cachefile) { - eval { $cache = retrieve($cachefile); }; - if ($@) { - warn "Problem loading $cachefile: $@"; - } - } - return $cache; -} - -sub store_obs_cache { - my ($self, $letter, $cache) = @_; - my $data = $self->data; - mkdir "$data/obs-cache" unless -d "$data/obs-cache"; - my $cachefile = "$data/obs-cache/$letter"; - store $cache, $cachefile; -} - -sub create_package_xml { - my ($self, $pkg, $spec) = @_; - my $xmlfile = "tmp-package.xml"; - my $xml = qq{<description/>}; - - if (-f $spec) { - my $noarch; - open my $fh, '<', $spec; - while (<$fh>) { - $noarch = 1, last if m/^BuildArch.*noarch/; - } - close $fh; - - if ($noarch) { - $xml .= q{<build><disable arch='i586'/></build>}; - } - } - else { - $xml .= q{<build><disable/></build>}; - } - $xml .= qq{</package>\n}; - { - open my $fh, '>', $xmlfile; - print $fh $xml; - close $fh; - } - return $xmlfile; -} - -sub update_obs { - my ($self, $letter, $args) = @_; - my $packages = $args->{packages}; - $self->lockdata; - my $data = $self->data; - - my $osc = "$data/osc"; - my $dir = "$osc/$letter"; - # removing previous osc checkouts - remove_tree $dir, { verbose => 0, safe => 1 }; - - my $states = $self->fetch_status($letter); - my @keys = sort keys %$states; - if ($packages and @$packages) { - info("Requested (@$packages)"); - @keys = @$packages; - } - - my $max = $args->{max}; - my $counter = 0; - for my $dist (@keys) { - my $dist_status = $states->{ $dist }; - unless ($dist_status) { - info("No status found for '$dist'"); - next; - } - my ($status, $version, $url) = @$dist_status; - unless ($args->{redo}) { - next if ($status eq 'done'); - } - if ($status =~ m/^error/) { - info("Skip $dist ($status)"); - next; - } - $counter++; - last if $counter > $max; - info("($counter) updating $dist (@$dist_status)"); - my $answer = 'Y'; - if ($args->{ask}) { - $answer = prompt("Update $dist? [y/N/q] ") || 'N'; - last if $answer eq 'Q'; - next if $answer eq 'N'; - info("y/n/q") if $answer ne 'Y'; - } - if ($answer eq 'Y') { - eval { - $self->osc_update_dist($letter, $dist, $dist_status, $args); - }; - if (my $err = $@) { - debug("ERROR: $dist $err"); - $states->{ $dist }->[0] = 'error'; - info("updating states ($letter)"); - $self->write_status($letter, $states); - } - } - } - $self->unlockdata; -} - -sub update_obs_perl { - my ($self, $args) = @_; - my $packages = $args->{packages}; - $self->lockdata; - my $project_prefix = $self->project_prefix; - my $data = $self->data; - my $apiurl = $self->apiurl; - - my $auto_projects = "$data/auto.xml"; - my $cmd = sprintf "osc -A $apiurl api /source/%s > %s", - $project_prefix, $auto_projects; - debug("CMD $cmd"); - system $cmd; - my $existing = XMLin($auto_projects)->{entry}; - - my $osc = "$data/osc"; - my $dir = "$osc/perl"; - # removing previous osc checkouts - remove_tree $dir, { verbose => 0, safe => 1 }; - - my $states = $self->fetch_status_perl(); - my @keys = sort keys %$states; - if ($packages and @$packages) { - info("Requested (@$packages)"); - @keys = @$packages; - } - - my $max = $args->{max}; - my $counter = 0; - for my $dist (@keys) { - my %args = %$args; - my $dist_status = $states->{ $dist }; - unless ($dist_status) { - info("No status found for '$dist'"); - next; - } - my ($status, $version, $url) = @$dist_status; - unless ($args{redo}) { - next if ($status eq 'done' or $status eq 'older'); - } - if ($status =~ m/^error/) { - info("Skip $dist ($status)"); - next; - } - - my $pkg = "perl-$dist"; - if ($existing->{ $pkg }) { - $args{exists} = 1; - my $tar = basename $url; - - my $pxml = "/tmp/$pkg-autoupdate.xml"; - my $cmd = sprintf "osc -A $apiurl api /source/%s/%s >%s", - $project_prefix, $pkg, $pxml; - system $cmd and die "Error ($cmd): $?"; - my $info = XMLin($pxml); - unlink $pxml; - if ($info->{entry}->{ $tar }) { - debug("$tar already exists, skipping"); - next; - } - } - - $counter++; - last if $counter > $max; - - info("($counter) updating $dist (@$dist_status)"); - my $answer = 'Y'; - if ($args{ask}) { - $answer = prompt("Update $dist? [y/N/q] ") || 'N'; - last if $answer eq 'Q'; - next if $answer eq 'N'; - info("y/n/q") if $answer ne 'Y'; - } - if ($answer eq 'Y') { - eval { - $self->osc_update_dist_perl($dist, $dist_status, \%args); - }; - my $err = $@; - if ($err) { - debug("ERROR: $dist $err"); - $states->{ $dist }->[0] = 'error'; - info("updating states (perl)"); - $self->write_status(perl => $states); - } - } - } - $self->unlockdata; -} - -sub osc_update_dist { - my ($self, $letter, $dist, $todo, $args) = @_; - my $data = $self->data; - my $apiurl = $self->apiurl; - my $mirror = $self->cpanmirror; - my $cpanspec = $self->cpanspec; - my $project_prefix = $self->project_prefix . $letter; - - my $osc = "$data/osc"; - mkdir $osc unless -d $osc; - my $dir = "$osc/$letter"; - mkdir $dir unless -d $dir; - chdir $dir; - debug("osc_update_dist($dist)"); - my ($status, $version, $url, $obs_tar, $obs_status) = @$todo; - my $pkg = "perl-$dist"; - my $spec = "$pkg.spec"; - my $tar = basename $url; - - { - my $cmd = "wget --tries 5 --timeout 30 --connect-timeout 30 -nc -q $mirror/authors/id/$url -O $tar -o /dev/null"; - debug("CMD $cmd"); - system $cmd; - if ($? or not -f $tar) { - info("Error fetching $url, skip ('$cmd': $?)"); - return 0; - } - } - my $error = 1; - { - my $cmd = sprintf - "timeout 180 perl $cpanspec -v -f --pkgdetails %s --skip-changes %s > cpanspec.error 2>&1", - "$data/02packages.details.txt.gz", $tar; - debug("CMD $cmd"); - if (system $cmd or not -f $spec) { - info("Error executing cpanspec"); - system("cat cpanspec.error"); - } - else { - system("cat cpanspec.error"); - $error = 0; - unlink "cpanspec.error"; - } - } - my $xmlfile = $self->create_package_xml($pkg, $spec); - my $project = "$project_prefix/$pkg"; - - { - my $cmd = sprintf "osc -A %s meta pkg %s %s -F %s", - $apiurl, $project_prefix, $pkg, $xmlfile; - debug("CMD $cmd"); - system $cmd - and die "Error executing '$cmd': $?"; - } - - my $checkout = "$dir/$project_prefix/$pkg"; - if (-e $checkout) { - debug("REMOVE $checkout"); - remove_tree $checkout, { verbose => 0, safe => 1 }; - } - { - my $cmd = sprintf "osc -A %s co %s/%s", - $apiurl, $project_prefix, $pkg; - system $cmd - and die "Error executing '$cmd': $?"; - } - chdir $checkout; - if ($obs_tar) { - my $cmd = "[[ -e $obs_tar ]] && rm $obs_tar || true"; - debug("CMD $cmd"); - system $cmd and die "Error executig '$cmd': $?"; - - } - { - my $cmd = "[[ -e cpanspec.error ]] && rm cpanspec.error || true"; - debug("CMD $cmd"); - system $cmd and die "Error executig '$cmd': $?"; - - $cmd = "[[ -e $spec ]] && rm $spec || true"; - debug("CMD $cmd"); - system $cmd and die "Error executig '$cmd': $?"; - } - if ($error) { - move "../../cpanspec.error", $checkout or die $!; - } - else { - move "../../$spec", $checkout or die $!; - } - move "../../$tar", $checkout or die $!; - { - my $cmd = "osc addremove"; - debug("CMD $cmd"); - system $cmd and die "Error executig '$cmd': $?"; - if ($args->{ask_commit}) { - my $answer = prompt("Commit? [Y/n]") || 'Y'; - if ($answer ne 'Y') { - info("$pkg - no commit"); - return 1; - } - } - - $cmd = "osc ci -mupdate"; - debug("CMD $cmd"); - system $cmd and die "Error executig '$cmd': $?"; - } - unlink "$dir/$xmlfile"; -} - -sub osc_update_dist_perl { - my ($self, $dist, $todo, $args) = @_; - my $exists = $args->{exists}; - my $data = $self->data; - my $apiurl = $self->apiurl; - my $mirror = $self->cpanmirror; - my $cpanspec = $self->cpanspec; - my $project_prefix = $self->project_prefix; - my $letter = 'perl'; - - my $osc = "$data/osc"; - mkdir $osc unless -d $osc; - my $dir = "$osc/$letter"; - mkdir $dir unless -d $dir; - chdir $dir; - debug("osc_update_dist($dist)"); - my ($status, $version, $url, $obs_tar, $obs_status) = @$todo; - my $pkg = "perl-$dist"; - my $spec = "$pkg.spec"; - my $tar = basename $url; - - if ($exists) { - my $cmd = sprintf "osc -A $apiurl rdelete -mrecreate -f %s %s", - $project_prefix, $pkg; - debug("CMD $cmd"); - system $cmd and die "Error ($cmd): $?"; - } - - { - my $cmd = sprintf "osc -A $apiurl branch devel:languages:perl %s %s", - $pkg, $project_prefix; - debug("CMD $cmd"); - system $cmd and die "Error ($cmd): $?"; - } - { - my $cmd = sprintf - "wget --tries 5 --timeout 30 --connect-timeout 30 -nc -q %s -O $dir/$tar -o /dev/null", - "$mirror/authors/id/$url"; - debug("CMD $cmd"); - system $cmd; - if ($? or not -f "$dir/$tar") { - info("Error fetching $url, skip ('$cmd': $?)"); - return 0; - } - } - - my $checkout = "$dir/$project_prefix/$pkg"; - if (-e $checkout) { - debug("REMOVE $checkout"); - remove_tree $checkout, { verbose => 0, safe => 1 }; - } - { - my $cmd = sprintf "osc -A %s co %s/%s", - $apiurl, $project_prefix, $pkg; - system $cmd - and die "Error executing '$cmd': $?"; - } - chdir $checkout; - - my $old_tar = ''; - for my $tar (bsd_glob("{$dist-*.tar*,$dist-*.tgz,$dist-*.zip}")) { - $old_tar = $tar; - unlink $tar; - } - move "$dir/$tar", $checkout or die $!; - my $error = 1; - copy("$Bin/../cpanspec.yml", "$checkout/cpanspec.yml") unless -f "cpanspec.yml"; - { - my $cmd = sprintf - "timeout 900 perl $cpanspec -v -f --pkgdetails %s --old-file %s %s > cpanspec.error 2>&1", - "$data/02packages.details.txt.gz", ".osc/$old_tar", $tar; - debug("CMD $cmd"); - if (system $cmd or not -f $spec) { - system("cat cpanspec.error"); - info("Error executing cpanspec"); - } - else { - system("cat cpanspec.error"); - $error = 0; - unlink "cpanspec.error"; - } - } - - { - my $cmd = "osc addremove"; - debug("CMD $cmd"); - system $cmd and die "Error executig '$cmd': $?"; - if ($args->{ask_commit}) { - my $answer = prompt("Commit? [Y/n]") || 'Y'; - if ($answer ne 'Y') { - info("$pkg - no commit"); - return 1; - } - } - - $cmd = "osc ci -mupdate"; - debug("CMD $cmd"); - system $cmd and die "Error executig '$cmd': $?"; - } -} - -sub update_status { - my ($self, $letter) = @_; - $self->lockdata; - - my $states = $self->fetch_status($letter); - my $upstream = $self->from_cpan_file($letter); - my $obs_info = $self->fetch_obs_info($letter); - for my $dist (sort keys %$upstream) { - my $pkg = "perl-$dist"; - my $info = $upstream->{ $dist }; - my $dist_status = $states->{ $dist }; - my $obs_ok = $obs_info->{ $pkg }->{ok} // 0; - my $obs_tar = $obs_info->{ $pkg }->{archive} || ''; - - my ($upstream_version, $upstream_url) = @$info; - my $upstream_tar = basename $upstream_url; - - my $status = 'new'; - if ($dist_status) { - $status = $dist_status->[0]; - - if ($status =~ m/^error/) { - } - elsif (not $obs_tar) { - $status = 'new'; - } - elsif ($obs_tar eq $upstream_tar) { - $status = 'done'; - } - else { - $status = 'todo'; - } - } - else { - if (not $obs_tar) { - $status = 'new'; - } - elsif ($obs_tar eq $upstream_tar) { - $status = 'done'; - } - else { - $status = 'todo'; - } - } - $dist_status = [ - $status, $upstream_version, $upstream_url, $obs_tar, $obs_ok, - ]; - - $states->{ $dist } = $dist_status; - - } - $self->write_status($letter, $states); - - $self->unlockdata; -} - -sub update_status_perl { - my ($self) = @_; - $self->lockdata; - my $apiurl = $self->apiurl; - my $data = $self->data; - - my $perl_projects = "$data/perl.xml"; - my $cmd = "osc -A $apiurl api /status/project/devel:languages:perl > $perl_projects"; - debug("CMD $cmd"); - system $cmd; - my $existing = XMLin($perl_projects)->{package}; - - my $states = $self->fetch_status_perl(); - my $upstream = {}; - for my $letter ('A' .. 'Z') { - my $up = $self->from_cpan_file($letter); - %$upstream = ( %$upstream, %$up ); - } - for my $dist (sort keys %$upstream) { - my $pkg = "perl-$dist"; - next unless defined $existing->{ $pkg }; - my $ex = $existing->{ $pkg }; - my $ex_version = $ex->{version}; - my $ex_version_normal = eval { version->parse($ex_version || 0) }; - next unless $ex_version_normal; - my $info = $upstream->{ $dist }; - my $dist_status = $states->{ $dist }; - - my ($upstream_version, $upstream_url) = @$info; - my $upstream_version_normal = eval { - version->parse($upstream_version) - }; - my $upstream_tar = basename $upstream_url; - - my $status = 'new'; - if ($dist_status) { - $status = $dist_status->[0]; - } - - if ($status =~ m/^error/) { - } - elsif (not $ex_version) { - $status = 'new'; - } - elsif ($ex_version_normal == $upstream_version_normal) { - $status = 'done'; - } - elsif ($ex_version_normal > $upstream_version_normal) { - $status = 'older'; - } - else { - $status = 'todo'; - } - - $dist_status = [ - $status, $upstream_version, $upstream_url, $ex_version_normal, - ]; - - $states->{ $dist } = $dist_status; - - } - $self->write_status('perl', $states); - - $self->unlockdata; -} - -sub lockdata { - my ($self) = @_; - my $data = $self->data; - mkdir $data unless -d $data; - return 1 if $self->locked; - - my $lockfile = "$data/lockfile"; - my $cmd = "lockfile -1 -r 5 $lockfile"; - system $cmd - and die "Error using lockfile: $?. Is another process running?"; - - $self->locked(1); -} - -sub unlockdata { - my ($self) = @_; - return 1 unless $self->locked; - - my $data = $self->data; - my $lockfile = "$data/lockfile"; - unlink $lockfile or die $!; - - $self->locked(0); -} - -1; diff --git a/munin/build-status-cron.sh b/munin/build-status-cron.sh deleted file mode 100755 index 8d6deab..0000000 --- a/munin/build-status-cron.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -# Writes OBS build statistics to YAML files - -# crontab example: -# 51,11,31 * * * * STAT_DIR=$HOME/munin/obs-cpan BUILD_STATUS=/path/to/cpanspec/bin/build-status /path/to/build-status-cron.sh >>$HOME/munin/build-status-cron.log 2>&1 - -#STAT_DIR=$HOME/munin/obs-cpan -#BUILD_STATUS=$HOME/develop/github/cpanspec/bin/build-status - -case "$1" in - -h|--help) - echo "Wrapper for the various build-status scripts" - exit - ;; -esac - -if [[ -z "$STAT_DIR" ]]; then - echo "Set STAT_DIR=~/path/to/stats" >&2 - exit 1 -fi -if [[ -z "$BUILD_STATUS" ]]; then - echo "Set BUILD_STATUS=/path/to/cpanspec/bin/build-status" >&2 - exit 1 -fi - - -[[ ! -d $STAT_DIR ]] && mkdir $STAT_DIR - -cd $STAT_DIR - -for LETTER in A B C D E F G H I J K L M N O P Q R S T U V W X Y Z; do - $BUILD_STATUS \ - --project-prefix devel:languages:perl:CPAN- --yaml "$LETTER" \ - >/tmp/build-status.yaml \ - && mv /tmp/build-status.yaml $STAT_DIR/build-status-$LETTER.yaml - sleep 1 -done - -$BUILD_STATUS \ - --project devel:languages:perl --yaml perl \ - >/tmp/build-status.yaml \ - && mv /tmp/build-status.yaml $STAT_DIR/build-status-perl.yaml - -$BUILD_STATUS \ - --project devel:languages:perl:autoupdate --yaml autoupdate \ - --repo standard \ - >/tmp/build-status.yaml \ - && mv /tmp/build-status.yaml $STAT_DIR/build-status-autoupdate.yaml diff --git a/munin/munin-build-status b/munin/munin-build-status deleted file mode 100755 index ef2e137..0000000 --- a/munin/munin-build-status +++ /dev/null @@ -1,119 +0,0 @@ -#!/usr/bin/env perl - -# Munin configuration -# put the following into /etc/munin/plugin-conf.d/munin-node -# -# [devel-build-status-*] -# env.datadir /home/user/munin/obs-cpan -# -# Create plugin scripts: -# cd /etc/munin/plugins -# for i in A B C D E F G H I J K L M N O P Q R S T U V W X Y Z autoupdate perl total; -# do -# ln -s /path/to/cpanspec/munin/munin-build-status devel-build-status-$i -# done -# -# Then restart munin-node. -# -# You can manually call a script like this to see what data munin will get: -# /etc/munin/plugins/devel-build-status-A -# /etc/munin/plugins/devel-build-status-perl -# /etc/munin/plugins/devel-build-status-autoupdate - - -use strict; -use warnings; -use 5.010; -use Data::Dumper; -use YAML::XS qw/ LoadFile /; - -my $script = $0; -my $path = $ENV{datadir}; - -if (@ARGV and $ARGV[0] =~ m/^(-h|--help)$/) { - say "Script to be called by munin, see munin documentation"; - exit; -} - -my @states = qw/ - building finished scheduled blocked broken - succeeded failed unresolvable disabled excluded -/; - -my $letter; -if ($script =~ m/-([A-Z])\z/) { - $letter = $1; -} -elsif ($script =~ m/-total\z/) { - $letter = 'total'; -} -elsif ($script =~ m/-perl\z/) { - $letter = 'perl'; -} -elsif ($script =~ m/-autoupdate\z/) { - $letter = 'autoupdate'; -} -else { - die "script '$0' not supported"; -} - -if ( defined $ARGV[0] and $ARGV[0] eq "config" ) { - config(); - exit; -} - -my $data = {}; - -if ($letter eq 'total') { - for my $letter ('A' .. 'Z') { - my $file = "$path/build-status-$letter.yaml"; - my $letter_data = LoadFile($file); - $letter_data = $letter_data->{ $letter }; - for my $state (@states) { - $data->{ $state } += $letter_data->{ $state } || 0; - } - } -} -else { - my $file = "$path/build-status-$letter.yaml"; - $data = LoadFile($file); - $data = $data->{ $letter }; -} - - -{ - my $total = 0; - for my $state (@states) { - my $value = $data->{ $state } || 0; - $total += $value; - print <<"EOM"; -$state.value $value -EOM - } - print <<"EOM"; -total.value $total -EOM -} - - -#graph_total total -sub config { - print <<"EOM"; -graph_title CPAN Mirror Build Status $letter -graph_args --base 1000 -l 0 -graph_category obs-cpan -graph_order succeeded unresolvable failed building finished scheduled blocked broken disabled excluded -graph_vlabel packages -EOM -for my $state (@states) { - print <<"EOM"; -$state.label $state -$state.draw LINE -EOM -} -print <<"EOM"; -total.label Total -total.graph no -EOM - -} diff --git a/updateallcpan b/updateallcpan deleted file mode 100755 index dfc9f62..0000000 --- a/updateallcpan +++ /dev/null @@ -1,304 +0,0 @@ -#! /usr/bin/perl - -# Copyright (C) 2015-2016 Stephan Kulow <coolo@suse.com> - -# This program is free software; you can redistribute it -# and/or modify it under the same terms as Perl itself. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. - -use strict; -use warnings; -use XML::Simple; -use Data::Dumper; -use Compress::Zlib; -use version; -use List::Util; -use File::Basename; -use Storable; -use Parse::CPAN::Packages; - -require CPAN::Meta::Requirements; -use File::Temp qw/tempdir tempfile/; -use File::Copy qw/copy/; - -my $scriptdir; - -#my $mirror = 'ftp://cpan.mirror.iphh.net/pub/CPAN'; -my $mirror = 'http://cpan.noris.de'; - -BEGIN { - ($scriptdir) = $0 =~ m-(.*)/-; - $scriptdir ||= '.'; - use Cwd 'abs_path'; - $scriptdir = abs_path($scriptdir); -} - -my $modpath = glob("~/.cpan/sources/modules"); -my $details = "02packages.details.txt.gz"; -chdir($modpath) || die "call cpan once to create $modpath"; -system("wget -qm -nd http://www.cpan.org/modules/$details"); - -my %upstream; - -my $letter = $ARGV[0]; -my %lpkgs; -my $cache = {}; -eval { $cache = retrieve("$scriptdir/obs_cache"); }; - -my $xml; - -my %autoupdate; - -if ($letter) { - open(my $statusfh, "-|", "osc api /status/project/devel:languages:perl"); - $xml = XMLin($statusfh)->{entry}; - close($statusfh); - - open($statusfh, "-|", "osc api /source/devel:languages:perl:CPAN-$letter?view=info"); - my $info = XMLin($statusfh)->{sourceinfo}; - close($statusfh); - for my $pi (@$info) { - if (!defined $cache->{$pi->{srcmd5}}) { - # TODO: cache - print "CURL $pi->{package}\n"; - open(my $statusfh, "-|", "osc api /source/devel:languages:perl:CPAN-$letter/$pi->{package}"); - my $pxml = XMLin($statusfh); - close($statusfh); - for my $entry (keys %{$pxml->{entry}}) { - if ($entry =~ m/\.tar/ || $entry =~ m/\.tgz$/ || $entry =~ m/\.zip$/) { - my $name = $pxml->{name}; - $lpkgs{$name} = $entry; - } - } - $cache->{$pi->{srcmd5}} = $lpkgs{$pi->{package}} || ''; - store $cache, "$scriptdir/obs_cache"; - } - else { - $lpkgs{$pi->{package}} = $cache->{$pi->{srcmd5}}; - } - } -} -else { - open(my $statusfh, "-|", "osc api /status/project/devel:languages:perl"); - $xml = XMLin($statusfh)->{package}; - close($statusfh); - - open($statusfh, "-|", "osc api /source/devel:languages:perl:autoupdate"); - my $info = XMLin($statusfh)->{entry}; - close($statusfh); - for my $p (keys %$info) { - $autoupdate{$p} = 1; - } -} - -my %letter_todo; - -my $p = Parse::CPAN::Packages->new("02packages.details.txt.gz"); -my %seen_distris; -for my $m ($p->packages) { - $m = $m->distribution; - if (!$m->version) { - #printf "Distribution %s has no version defined\n", $m->prefix; - next; - } - next if $seen_distris{$m->prefix} && $seen_distris{$m->prefix}[0] eq $m->version; - next unless $m->dist; - my $version; - eval { $version = version->parse($m->version); }; - $version ||= version->declare('0'); - $seen_distris{$m->prefix} = [ $m->version, $version ]; - my $url = $m->prefix; - my $uversion; - $uversion = $upstream{$m->dist}[0] if defined $upstream{$m->dist}; - $uversion ||= version->declare('0'); - #printf "PKG $url $version %s - %s (%s vs %s)\n", $m->dist, $m->filename, $uversion->stringify, $version->stringify; - next if $version < $uversion; - $upstream{$m->dist} = [$version, $url]; - if ($letter) { - my $tar = $m->filename; - if ($tar && uc(substr($tar, 0, 1)) eq $letter) { - if (basename($tar, qw/.tar.gz .tgz .zip/) =~ m!^(\S+)-([^-]*)$!) { - my $pkg = $1; - while ($pkg =~ m/^(.*)-(v?[0-9][^-]*)$/) { - $pkg = $1; - } - next if defined $xml->{"perl-$pkg"}; - my $obs = $lpkgs{"perl-$pkg"} || ''; - next if $obs eq 'done'; - $letter_todo{$pkg} = [$tar, $obs, $url] if $obs ne $tar; - $lpkgs{"perl-$pkg"} = 'done'; - } - } - } -} - -if ($letter) { - for my $pkg (sort keys %letter_todo) { - - my ($tar, $obs, $url) = @{$letter_todo{$pkg}}; - my ($fh, $filename) = tempfile(); - print $fh "<package name='perl-$pkg'><title/><description/><build><disable/></build></package>\n"; - close($fh); - system("osc meta pkg devel:languages:perl:CPAN-$letter perl-$pkg -F $filename"); - if ($obs) { - print "osc api -X DELETE /source/devel:languages:perl:CPAN-$letter/perl-$pkg/$obs\n"; - system("osc api -X DELETE /source/devel:languages:perl:CPAN-$letter/perl-$pkg/$obs"); - } - my $tempdir = tempdir(CLEANUP => 1); - chdir($tempdir); - system("osc co devel:languages:perl:CPAN-$letter/perl-$pkg") == 0 - or goto CHDIR; - print "TAR $tar '$obs' '$pkg'\n"; - chdir("devel:languages:perl:CPAN-$letter/perl-$pkg") || die "can't chdir"; - #print "wget -nc -q $mirror/authors/id/$url\n"; - system("wget -nc -q $mirror/authors/id/$url"); - my $worked; - unlink("perl-$pkg.spec"); - if (system("$scriptdir/cpanspec -v -f --skip-changes $tar > cpanspec.error 2>&1") == 0 && -f "perl-$pkg.spec") { - unlink("cpanspec.error"); - $worked = 1; - } - system("osc addremove") == 0 - or goto CHDIR; - my $noarch; - if (-f "perl-$pkg.spec") { - open(SPEC, "perl-$pkg.spec"); - while (<SPEC>) { - $noarch = 1 if m/^BuildArch.*noarch/; - } - close(SPEC); - } - ($fh, $filename) = tempfile(); - if ($worked) { - if ($noarch) { - print $fh "<package name='perl-$pkg'><title/><description/><build><disable arch='i586'/></build></package>\n"; - print "build disable i586\n"; - } - else { - print $fh "<package name='perl-$pkg'><title/><description/></package>\n"; - print "build enable all\n"; - } - close($fh); - #print "osc meta pkg devel:languages:perl:CPAN-$letter perl-$pkg -F $filename\n"; - system("osc meta pkg devel:languages:perl:CPAN-$letter perl-$pkg -F $filename"); - } - system("osc ci -mupdate"); - CHDIR: # finally emulation - chdir("/tmp"); - } - - for my $pkg (sort keys %lpkgs) { - next if $lpkgs{$pkg} eq 'done'; - print "osc rdelete -mgone devel:languages:perl:CPAN-$letter $pkg\n"; - system "osc rdelete -mgone devel:languages:perl:CPAN-$letter $pkg"; - } - exit(0); -} - -my $tocreate = 3000; - -my @pkgs = List::Util::shuffle(keys %$xml); -@pkgs = sort keys %$xml; -my %tobuild; -while ((@pkgs && $tocreate) || %tobuild) { - if (%tobuild) { - my $url = "/build/devel:languages:perl:autoupdate/_result?"; - my @tocheck = sort keys %tobuild; - $url .= "package=" . shift @tocheck; - $url .= "&package=$_" for (@tocheck); - print "checking '$url'\n"; - open(my $fh, "-|", "osc api '$url'"); - my $res = XMLin($fh, forcearray => [qw/status/]); - #print Dumper($res); - close($fh); - - if ($res && $res->{result}->{status} && ($res->{result}->{dirty} || '') ne 'true') { - #print Dumper($res); - for my $status (@{$res->{result}->{status}}) { - my $code = $status->{code} || 'unscheduled'; - if ($code && $code eq 'finished') { - $code = $status->{details} || 'unknown'; - } - my $built = $status->{package}; - print "CODE $built $code\n"; - if ($code eq 'succeeded') { - system("osc sr -m 'automatic update' devel:languages:perl:autoupdate $built devel:languages:perl --clean < /dev/null"); - delete $tobuild{$built}; - } - if ($code eq 'failed' || $code eq 'unresolvable' || $code eq 'broken') { - delete $tobuild{$built}; - } - } - } - } - if ($tocreate && @pkgs) { - my $pkg = shift @pkgs; - next unless $pkg =~ m,^perl-,; - my $ups = $pkg; - $ups =~ s,^perl-,,; - if (!defined($upstream{$ups}[0])) { - print "echo osc rdelete -mgone devel:languages:perl/$pkg\n"; - } - next unless defined($upstream{$ups}[0]); - my $obs_version; - eval { $obs_version = version->parse($xml->{$pkg}->{version}); }; - next unless $obs_version; - my $older = $obs_version < $upstream{$ups}[0]; - if (!$older) { - system("osc rdelete -mfresh -f devel:languages:perl:autoupdate $pkg") if $autoupdate{$pkg}; - #print "PKG $pkg " . $obs_version . " $ups " . $upstream{$ups}[0] . " NEWER\n"; - next; - } - print "PKG $pkg " . $obs_version . " $ups " . $upstream{$ups}[0] . " " . ($older ? 'OLDER' : 'NEWER') . "\n"; - my $ntar = basename($upstream{$ups}[1]); - open(my $statusfh, "-|", "osc api /source/devel:languages:perl/$pkg"); - my $pxml = XMLin($statusfh); - close($statusfh); - if ($pxml->{entry}->{$ntar}) { - #print "ALREADY THERE: $pkg\n"; - system("osc rdelete -mfresh -f devel:languages:perl:autoupdate $pkg") if $autoupdate{$pkg}; - next; - } - my $tempdir = tempdir; # ( CLEANUP => 1 ); - chdir($tempdir); - - if (!$autoupdate{$pkg} && system("osc branch devel:languages:perl $pkg devel:languages:perl:autoupdate") != 0) { - print "branch of $pkg failed\n"; - next; - } - my $retries = 0; - # we need to keep trying because the service needs to finish first - while (system("osc co devel:languages:perl:autoupdate $pkg")) { - print "checkout failed: $retries\n"; - last if ($retries++ == 3); - sleep(3); - } - next if ($retries > 2); - $tocreate--; - print "branched\n"; - chdir("devel:languages:perl:autoupdate/$pkg") || die "can't chdir"; - for my $tar (glob("*.tar*")) { - unlink($tar); - } - system("wget -q $mirror/authors/id/$upstream{$ups}[1]"); - copy("$scriptdir/cpanspec.yml", "cpanspec.yml") unless -f "cpanspec.yml"; - system("$scriptdir/cpanspec -f --old-file .osc/*.tar* *.tar*") == 0 - or next; - system("osc addremove") == 0 - or next; - print "update\n"; - system("osc ci -mupdate") == 0 - or next; - $tobuild{$pkg} = 1; - print "SHELL $tempdir\n"; - chdir("/tmp"); - } - else { - sleep(5); - } - #exit(1); - #exit(1) if ($requests++ > 5); -}