Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

prevent inserts of existing entries, update to support scripts

  • Loading branch information...
commit db4bf63048f7e66f59a364d23d61177474295629 1 parent d09eecf
barbie authored
View
5 CHANGES
@@ -1,5 +1,9 @@
Revision history for Perl module CPAN::Testers::Data::Generator.
+0.33 29/12/2008
+ - updated help text to bin/cpanstats-verify (v0.23)
+ - fix to stop existing records being reinserted.
+
0.32 19/12/2008
- added progress option to bin/cpanstats-verify (v0.22)
- fixes to Article.pm to attempt to correct missing osnames and osvers.
@@ -7,7 +11,6 @@ Revision history for Perl module CPAN::Testers::Data::Generator.
sometime:
bin/articles-verify (v0.02)
bin/articles-view (v0.02)
-
0.31 18/12/2008
- name change to fit below the CPAN::Testers namespace.
- added epoch to date extraction from article.
View
6 META.yml
@@ -1,6 +1,6 @@
--- #YAML:1.0
name: CPAN-Testers-Data-Generator
-version: 0.32
+version: 0.33
abstract: Download and summarize CPAN Testers data
author:
- Barbie <barbie@cpan.org>
@@ -38,10 +38,10 @@ build_requires:
provides:
CPAN::Testers::Data::Generator:
file: lib/CPAN/Testers/Data/Generator.pm
- version: 0.32
+ version: 0.33
CPAN::Testers::Data::Generator::Article:
file: lib/CPAN/Testers/Data/Generator/Article.pm
- version: 0.32
+ version: 0.33
no_index:
directory:
View
7 bin/cpanstats-select
@@ -80,7 +80,7 @@ if($options{sql}) {
print join(",",@$row) . "\n";
}
} else {
- print "Sorry, no results returned\n";
+ print "Sorry, no results returned for SQL [$options{sql}]\n";
}
} else {
my $sql = "SELECT * FROM cpanstats WHERE ";
@@ -103,13 +103,14 @@ if($options{sql}) {
push @where, "perl=''" if(defined $options{perl} && $options{perl} eq '-');
if(@where) {
- my @rows = $dbi->get_query('array',$sql . join(' AND ',@where));
+ $sql .= join(' AND ',@where);
+ my @rows = $dbi->get_query('array',$sql);
if(@rows) {
for my $row (@rows) {
print join(",",@$row) . "\n";
}
} else {
- print "Sorry, no results returned\n";
+ print "Sorry, no results returned for SQL [$sql]\n";
}
} else {
print "No SQL arguments given\n";
View
10 bin/cpanstats-verify
@@ -2,7 +2,7 @@
use strict;
$|++;
-my $VERSION = '0.22';
+my $VERSION = '0.23';
#----------------------------------------------------------------------------
@@ -436,16 +436,18 @@ sub help {
if($full) {
print <<HERE;
-
Usage: $0 \\
--config=<file> --database=<dbcode> \\
+ --log=<file> --out=<file> \\
[-c] [-m] [-v] [-s] \\
[--file=<file>] [--start=n] [--end=n] \\
- [--log=<file>] [-out=<file>] [-h] [-V] \\
+ [-h] [-V] \\
[--localonly] [--progress]
--config=<file> - configuration file
--database=<dbcode> - CPANSTATS or LITESTATS
+ --log - log file for shortcut reference for -m and -c
+ --out - results output file
-m - check for missing entries
-c - check existing entries for bad parsing
@@ -454,8 +456,6 @@ Usage: $0 \\
--file - file of IDs to reference
--start - start id for -m or -c
--end - end id for -m or -c
- --log - log file for shortcut reference for -m and -c
- --out - results output file
--localonly - no NNTP lookups, use local logs only
--progress - print progress messages to STDOUT
View
17 lib/CPAN/Testers/Data/Generator.pm
@@ -4,7 +4,7 @@ use warnings;
use strict;
use vars qw($VERSION);
-$VERSION = '0.32';
+$VERSION = '0.33';
#----------------------------------------------------------------------------
# Library Modules
@@ -264,8 +264,12 @@ sub insert_stats {
my $INSERT = 'INSERT INTO cpanstats VALUES (?,?,?,?,?,?,?,?,?,?,?)';
- $self->{CPANSTATS}->do_query($INSERT,@fields);
- $self->{LITESTATS}->do_query($INSERT,@fields);
+ for my $db (qw(CPANSTATS LITESTATS)) {
+ my @rows = $self->{$db}->get_query('array','SELECT * FROM cpanstats WHERE id=?',$fields[0]);
+ next if(@rows);
+ $self->{$db}->do_query($INSERT,@fields);
+ }
+
if((++$self->{stat_count} % 50) == 0) {
$self->{CPANSTATS}->do_commit;
$self->{LITESTATS}->do_commit;
@@ -281,7 +285,12 @@ sub insert_article {
my $INSERT = 'INSERT INTO articles VALUES (?,?)';
- $self->{LITEARTS}->do_query($INSERT,@fields);
+ for my $db (qw(LITEARTS)) {
+ my @rows = $self->{$db}->get_query('array','SELECT * FROM articles WHERE id=?',$fields[0]);
+ next if(@rows);
+ $self->{$db}->do_query($INSERT,@fields);
+ }
+
if((++$self->{arts_count} % 50) == 0) {
$self->{LITEARTS}->do_commit;
}
View
2  lib/CPAN/Testers/Data/Generator/Article.pm
@@ -4,7 +4,7 @@ use warnings;
use strict;
use vars qw($VERSION);
-$VERSION = '0.32';
+$VERSION = '0.33';
#----------------------------------------------------------------------------
# Library Modules
Please sign in to comment.
Something went wrong with that request. Please try again.