Skip to content

Commit

Permalink
added guid support
Browse files Browse the repository at this point in the history
  • Loading branch information
@barbie
barbie committed Jan 30, 2010
1 parent abab715 commit e6f246a
Show file tree
Hide file tree
Showing 14 changed files with 131 additions and 86 deletions.
3 changes: 3 additions & 0 deletions CHANGES
View file
@@ -1,5 +1,8 @@
Revision history for Perl module CPAN::Testers::Data::Generator.

0.39 30/01/2010
- major update to include guid for Metabase/CT2.0.

0.38 19/10/2009
- added functionality to update the osname table.
- added cpanstats DB schema to examples directory.
Expand Down
5 changes: 3 additions & 2 deletions META.yml
View file
@@ -1,6 +1,6 @@
--- #YAML:1.0
name: CPAN-Testers-Data-Generator
version: 0.38
version: 0.39
abstract: Download and summarize CPAN Testers data
author:
- Barbie <barbie@cpan.org>
Expand All @@ -14,6 +14,7 @@ requires:
Config::IniFiles: 0
CPAN::Testers::Common::Article: 0.36
CPAN::Testers::Common::DBUtils: 0.03
CPAN::Testers::Common::Utils: 0.01
DBD::mysql: 0
DBD::SQLite: 1.07
Email::Simple: 0
Expand All @@ -36,7 +37,7 @@ build_requires:
provides:
CPAN::Testers::Data::Generator:
file: lib/CPAN/Testers/Data/Generator.pm
version: 0.38
version: 0.39

no_index:
directory:
Expand Down
1 change: 1 addition & 0 deletions Makefile.PL
View file
Expand Up @@ -10,6 +10,7 @@ WriteMakefile(
'Config::IniFiles' => 0,
'CPAN::Testers::Common::Article' => 0.36,
'CPAN::Testers::Common::DBUtils' => 0.03,
'CPAN::Testers::Common::Utils' => 0.01,
'File::Basename' => 0,
'File::Path' => 0,
'IO::File' => 0,
Expand Down
49 changes: 27 additions & 22 deletions bin/articles-verify
View file
Expand Up @@ -2,7 +2,7 @@
use strict;
$|++;

my $VERSION = '0.03';
my $VERSION = '0.04';

#----------------------------------------------------------------------------

Expand Down Expand Up @@ -101,13 +101,13 @@ processing.
sub check_articles {
my @list = get_list();
for my $id (@list) {
my @rows = $dbi->get_query('array',"SELECT * FROM articles WHERE id=$id");
if(@rows) {
my ($xref) = ($rows[0]->[1] =~ /Xref:.*perl\.cpan\.testers:($id)\b/is);
print "$id,invalid\n" unless($xref);
} else {
print "$id,missing\n";
}
my @rows = $dbi->get_query('array',"SELECT * FROM articles WHERE id=?",$id);
if(@rows) {
my ($xref) = ($rows[0]->[1] =~ /Xref:.*perl\.cpan\.testers:($id)\b/is);
print "$id,invalid\n" unless($xref);
} else {
print "$id,missing\n";
}
}
}

Expand All @@ -121,7 +121,7 @@ missing.
sub verify_articles {
my @list = get_list();
for my $id (@list) {
my @rows = $dbi->get_query('array',"SELECT * FROM articles WHERE id=$id");
my @rows = $dbi->get_query('array',"SELECT * FROM articles WHERE id=?",$id);
if(@rows) {
my $subject = get_subject($rows[0]->[1]);
print "$id,found - $subject\n";
Expand All @@ -133,19 +133,22 @@ sub verify_articles {

=item update_articles
Refresh the articles database, updating any existing records and inserting
missing records.
=cut

sub update_articles {
$nntp = Net::NNTP->new("nntp.perl.org")
|| die "Cannot connect to nntp.perl.org";
($num, $first, $last) = $nntp->group("perl.cpan.testers");

my @list = get_list();
my @list = get_list($last);
for my $id (@list) {
my $article = get_article($id);
next unless($article);

my @rows = $dbi->get_query('array',"SELECT id FROM articles WHERE id = $id");
my @rows = $dbi->get_query('array',"SELECT id FROM articles WHERE id=?",$id);
if(@rows) {
$dbi->do_query("UPDATE articles SET article=? WHERE id=?",$article,$id);
} else {
Expand Down Expand Up @@ -194,12 +197,13 @@ Returns the list of NNTP ids from the named file.
=cut

sub get_list {
my $last = shift || -1;
my @list;

if($options{start}) {
my $end = $options{end} || get_lastid();
my $end = $options{end} || get_lastid() || $last;
@list = ($options{start} .. $end);
print STDERR "START: $options{start}\nEND: $options{end}\n";
print STDERR "START: $options{start}\nEND: $options{end}\n";

} elsif($options{file}) {
die "file [$options{file}] not found" unless(-f $options{file});
Expand All @@ -211,7 +215,7 @@ sub get_list {
push @list, $num;
}
$fh->close;
printf STDERR "FILE: %d ids found\n", scalar(@list);
printf STDERR "FILE: %d ids found\n", scalar(@list);

} else {
die "No start/end or file list specified\n"
Expand All @@ -228,7 +232,8 @@ Returns the last NNTP id recorded in the database.

sub get_lastid {
my @rows = $dbi->get_query('array',"SELECT MAX(id) FROM articles");
return $rows[0]->[0];
return $rows[0]->[0] if(@rows);
return;
}

=item init_options
Expand Down Expand Up @@ -276,9 +281,9 @@ sub help {
print <<HERE;
Usage: $0 \\
--config=<file> \\
[ ( --start=n [--end=n] | --file=<file> ) ] \\
[ --update | --verify | --check ] \\
--config=<file> \\
[ ( --start=n [--end=n] | --file=<file> ) ] \\
[ --update | --verify | --check ] \\
[-h] [-v]
--config=<file> - configuration file
Expand All @@ -287,9 +292,9 @@ Usage: $0 \\
--end - verify to id
--file=<file> - verify these ids (1 per line)
--update - update articles
--verify - verify articles
--check - check articles
--update - update articles
--verify - verify articles
--check - check articles
-h - this help screen
-v - program version
Expand Down Expand Up @@ -334,7 +339,7 @@ F<http://wiki.cpantesters.org/>
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions.
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
Expand Down
2 changes: 1 addition & 1 deletion bin/articles-view
View file
Expand Up @@ -166,7 +166,7 @@ F<http://wiki.cpantesters.org/>
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions.
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
Expand Down
2 changes: 1 addition & 1 deletion bin/cpanstats
View file
Expand Up @@ -137,7 +137,7 @@ F<http://wiki.cpantesters.org/>
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions.
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
Expand Down
2 changes: 1 addition & 1 deletion bin/cpanstats-delete
View file
Expand Up @@ -199,7 +199,7 @@ F<http://wiki.cpantesters.org/>
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions.
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
Expand Down
2 changes: 1 addition & 1 deletion bin/cpanstats-reparse
View file
Expand Up @@ -191,7 +191,7 @@ F<http://wiki.cpantesters.org/>
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions.
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
Expand Down
2 changes: 1 addition & 1 deletion bin/cpanstats-select
View file
Expand Up @@ -230,7 +230,7 @@ for Miss Barbell Productions L<http://www.missbarbell.co.uk>.
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions
All Rights Reserved.
This module is free software; you can redistribute it and/or
Expand Down
2 changes: 1 addition & 1 deletion bin/cpanstats-update
View file
Expand Up @@ -229,7 +229,7 @@ F<http://wiki.cpantesters.org/>
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions.
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
Expand Down
2 changes: 1 addition & 1 deletion bin/cpanstats-verify
View file
Expand Up @@ -503,7 +503,7 @@ F<http://wiki.cpantesters.org/>
=head1 COPYRIGHT AND LICENSE
Copyright (C) 2005-2009 Barbie for Miss Barbell Productions.
Copyright (C) 2005-2010 Barbie for Miss Barbell Productions.
This module is free software; you can redistribute it and/or
modify it under the same terms as Perl itself.
Expand Down
83 changes: 50 additions & 33 deletions examples/cpanstats.sql
View file
Expand Up @@ -2,48 +2,55 @@

DROP TABLE IF EXISTS cpanstats;
CREATE TABLE cpanstats (
id int(10) unsigned NOT NULL,
state varchar(32),
postdate varchar(8),
tester varchar(255),
dist varchar(255),
version varchar(255),
platform varchar(255),
perl varchar(255),
osname varchar(255),
osvers varchar(255),
fulldate varchar(32),
PRIMARY KEY (id)
id int(10) unsigned NOT NULL,
guid char(36) NOT NULL,
state varchar(32),
postdate varchar(8),
tester varchar(255),
dist varchar(255),
version varchar(255),
platform varchar(255),
perl varchar(255),
osname varchar(255),
osvers varchar(255),
fulldate varchar(32),
PRIMARY KEY (id,guid)
);

DROP TABLE IF EXISTS `release_data`;
CREATE TABLE `release_data` (
`dist` varchar(255) NOT NULL,
`version` varchar(255) NOT NULL,
`id` int(10) unsigned NOT NULL,
`oncpan` tinyint(4) default '0',
`distmat` tinyint(4) default '0',
`perlmat` tinyint(4) default '0',
`patched` tinyint(4) default '0',
`pass` int(10) default '0',
`fail` int(10) default '0',
`na` int(10) default '0',
`unknown` int(10) default '0'
dist varchar(255) NOT NULL,
version varchar(255) NOT NULL,
id int(10) unsigned NOT NULL,
guid char(36) NOT NULL,
oncpan tinyint(4) default 0,
distmat tinyint(4) default 0,
perlmat tinyint(4) default 0,
patched tinyint(4) default 0,
pass int(10) default 0,
fail int(10) default 0,
na int(10) default 0,
unknown int(10) default 0,
PRIMARY KEY (id,guid),
INDEX (dist,version)
);

DROP TABLE IF EXISTS release_summary;
CREATE TABLE release_summary (
dist varchar(255) NOT NULL,
version varchar(255) NOT NULL,
id int(10) unsigned NOT NULL,
guid char(36) NOT NULL,
oncpan tinyint(4) default 0,
distmat tinyint(4) default 0,
perlmat tinyint(4) default 0,
patched tinyint(4) default 0,
pass int(10) default 0,
fail int(10) default 0,
na int(10) default 0,
unknown int(10) default 0
unknown int(10) default 0,
PRIMARY KEY (id,guid),
INDEX (dist,version)
);

DROP TABLE IF EXISTS `uploads`;
Expand All @@ -53,34 +60,34 @@ CREATE TABLE `uploads` (
`dist` varchar(255) NOT NULL,
`version` varchar(255) NOT NULL,
`filename` varchar(255) NOT NULL,
`released` int(16) NOT NULL,
`released` int(16) NOT NULL,
PRIMARY KEY (`author`,`dist`,`version`)
) ENGINE=MyISAM;

DROP TABLE IF EXISTS `ixlatest`;
CREATE TABLE `ixlatest` (
`dist` varchar(255) NOT NULL,
`version` varchar(255) NOT NULL,
`released` int(16) NOT NULL,
`released` int(16) NOT NULL,
`author` varchar(32) NOT NULL,
`oncpan` tinyint(4) DEFAULT 0,
PRIMARY KEY (`dist`)
) ENGINE=MyISAM;

DROP TABLE IF EXISTS `summary`;
CREATE TABLE `summary` (
`type` varchar(8) NOT NULL,
`name` varchar(255) NOT NULL,
`lastid` int(10) unsigned NOT NULL,
`dataset` blob,
`type` varchar(8) NOT NULL,
`name` varchar(255) NOT NULL,
`lastid` int(10) unsigned NOT NULL,
`dataset` blob,
PRIMARY KEY (`type`,`name`)
) ENGINE=MyISAM;

DROP TABLE IF EXISTS `page_requests`;
CREATE TABLE `page_requests` (
`type` varchar(8) NOT NULL,
`name` varchar(255) NOT NULL,
`weight` int(2) unsigned NOT NULL
`type` varchar(8) NOT NULL,
`name` varchar(255) NOT NULL,
`weight` int(2) unsigned NOT NULL
) ENGINE=MyISAM;

DROP TABLE IF EXISTS osname;
Expand All @@ -91,3 +98,13 @@ CREATE TABLE osname (
PRIMARY KEY (id)
);



DROP TABLE IF EXISTS fact;
CREATE TABLE fact (
guid char(36) NOT NULL,
type varchar(36) NOT NULL,
meta blob NOT NULL,
content blob NOT NULL,
PRIMARY KEY (guid)
);
3 changes: 3 additions & 0 deletions examples/settings-example.ini
View file
Expand Up @@ -13,3 +13,6 @@ dbhost=localhost
dbuser=username
dbpass=password

[DISABLE]
LIST=<<IGNORE
IGNORE

0 comments on commit e6f246a

Please sign in to comment.