Permalink
Browse files

added Data::FlexSerializer functionality to compress reports

  • Loading branch information...
1 parent fbdef1e commit bea981510e6163f63da97f6b5aa122cecd25a85a @barbie committed Sep 4, 2013
Showing with 26 additions and 3 deletions.
  1. +4 −1 Changes
  2. +1 −0 META.json
  3. +1 −0 META.yml
  4. +1 −0 Makefile.PL
  5. +5 −0 README
  6. +14 −2 lib/CPAN/Testers/Data/Generator.pm
View
@@ -1,4 +1,7 @@
-Revision history for Perl module CPAN-Testers-Data-Generator.
+Revision history for CPAN-Testers-Data-Generator
+================================================
+
+ - added Data::FlexSerializer functionality to compress reports.
1.10 2013-05-12
- fixed skip test count.
View
@@ -31,6 +31,7 @@
"CPAN::Testers::Fact::TestSummary": "0",
"CPAN::Testers::Metabase::AWS": "0",
"CPAN::Testers::Report": "0",
+ "Data::FlexSerializer": "0",
"DateTime": "0",
"DateTime::Duration": "0",
"DBI": "0",
View
@@ -18,6 +18,7 @@ requires:
CPAN::Testers::Fact::TestSummary: 0
CPAN::Testers::Metabase::AWS: 0
CPAN::Testers::Report: 0
+ Data::FlexSerializer: 0
DateTime: 0
DateTime::Duration: 0
DBI: 0
View
@@ -23,6 +23,7 @@ WriteMakefile(
'CPAN::Testers::Fact::TestSummary' => 0,
'CPAN::Testers::Metabase::AWS' => 0,
'CPAN::Testers::Report' => 0,
+ 'Data::FlexSerializer' => 0,
'DateTime' => 0,
'DateTime::Duration' => 0,
'File::Basename' => 0,
View
5 README
@@ -15,6 +15,9 @@ The distribution requires the following modules:
CPAN::Testers::Fact::TestSummary
CPAN::Testers::Metabase::AWS
CPAN::Testers::Report
+ Data::FlexSerializer
+ DateTime
+ DateTime::Duration
File::Basename
File::Path
File::Slurp
@@ -35,6 +38,8 @@ The distribution requires the following modules:
Getopt::ArgvFile
+
+
INSTALLATION
To install this module, untar the tarball into the directory of choice then
@@ -13,6 +13,7 @@ use Config::IniFiles;
use CPAN::Testers::Common::Article;
use CPAN::Testers::Common::DBUtils;
#use Data::Dumper;
+use Data::FlexSerializer;
use DateTime;
use DateTime::Duration;
use File::Basename;
@@ -149,6 +150,11 @@ sub new {
return unless($self->{metabase} && $self->{librarian});
}
+ # reports are now stored in a compressed format
+ $self->{serializer} = Data::FlexSerializer->new(
+ detect_compression => 1,
+ );
+
return $self;
}
@@ -297,6 +303,8 @@ $self->_log("START sql=[$sql]\n");
next;
}
+ $row->{report} = $self->{serializer}->deserialize($row->{report});
+
$self->{report}{id} = $row->{id};
$self->{report}{guid} = $row->{guid};
$self->{report}{metabase} = decode_json($row->{report});
@@ -633,7 +641,8 @@ sub already_saved {
sub load_fact {
my ($self,$guid,$check) = @_;
my @rows = $self->{METABASE}->get_query('array','SELECT report FROM metabase WHERE guid=?',$guid);
- return $rows[0]->[0] if(@rows);
+
+ return $self->{serializer}->deserialize($rows[0]->[0]) if(@rows);
$self->_log(" ... no report [guid=$guid]\n") unless($check);
return;
@@ -936,8 +945,11 @@ sub cache_report {
return 1 if($self->{check});
return 1 if($self->{localonly});
+ my $json = encode_json($self->{report}{metabase};
+ my $data = $self->{serializer}->serialize($json);
+
$self->{METABASE}->do_query('INSERT IGNORE INTO metabase (guid,id,updated,report) VALUES (?,?,?,?)',
- $self->{report}{guid},$self->{report}{id},$self->{report}{updated},encode_json($self->{report}{metabase}));
+ $self->{report}{guid},$self->{report}{id},$self->{report}{updated},$data));
if((++$self->{meta_count} % 500) == 0) {
$self->{METABASE}->do_commit;

0 comments on commit bea9815

Please sign in to comment.